diff --git a/doorstop/core/builder.py b/doorstop/core/builder.py index 5669061d3..7c95e02cf 100644 --- a/doorstop/core/builder.py +++ b/doorstop/core/builder.py @@ -2,8 +2,15 @@ """Functions to build a tree and access documents and items.""" + import os +from traceback import format_exception + from typing import List, Optional +from collections.abc import Mapping, MutableMapping +from pathlib import Path +from dulwich import porcelain +from dulwich.repo import Repo from doorstop import common from doorstop.common import DoorstopError @@ -28,17 +35,23 @@ def build(cwd=None, root=None, request_next_number=None) -> Tree: :return: new :class:`~doorstop.core.tree.Tree` """ - documents: List[Document] = [] + + documents: list[Document] = [] # Find the root of the working copy cwd = cwd or os.getcwd() root = root or vcs.find_root(cwd) + # Remote parents inclusion + remote_documents: dict[str,str] = {} # Prefix, tag + remote_base_path: Path | None = Path(root, ".doorstop-remote") or None #TODO: make this configurable + + # Find all documents in the working copy log.info("looking for documents in {}...".format(root)) skip_file_name = ".doorstop.skip-all" if not os.path.isfile(os.path.join(root, skip_file_name)): - _document_from_path(root, root, documents) + _document_from_path(root, root, documents, remote_documents, remote_base_path) exclude_dirnames = {".git", ".venv", "venv"} if not os.path.isfile(os.path.join(root, skip_file_name)): for dirpath, dirnames, _ in os.walk(root, topdown=True): @@ -50,7 +63,7 @@ def build(cwd=None, root=None, request_next_number=None) -> Tree: if os.path.isfile(os.path.join(path, skip_file_name)): continue whilelist_dirnames.append(dirname) - _document_from_path(path, root, documents) + _document_from_path(path, root, documents, remote_documents, remote_base_path) dirnames[:] = whilelist_dirnames # Build the tree @@ -66,7 +79,7 @@ def build(cwd=None, root=None, request_next_number=None) -> Tree: return tree -def _document_from_path(path, root, documents): +def _document_from_path(path, root, documents:list[Document], remote_documents:dict[str,str], remote_base_path: Path): """Attempt to create and append a document from the specified path. :param path: path to a potential document @@ -80,12 +93,27 @@ def _document_from_path(path, root, documents): except DoorstopError: pass # no document in directory else: + if document.remote_parent_url and document.remote_parent_tag: + log.debug("""Document has remote parent + Remote information: + Remote Url: {} + Remote Tag: {}""".format( + document.remote_parent_url, + document.remote_parent_tag)) if document.skip: log.debug("skipped document: {}".format(document)) else: log.info("found document: {}".format(document)) + if document.remote_parent_url and document.remote_parent_tag: + _download_remote_parent(remote_base_path, + document.remote_parent_url, + document.remote_parent_tag, + documents, + remote_documents) documents.append(document) + # log.debug("Remote documents in {}".format(remote_base_path / document.prefix)) + def find_document(prefix): """Find a document without an explicitly building a tree.""" @@ -120,3 +148,111 @@ def _clear_tree(): """Force the shared tree to be rebuilt.""" global _tree _tree = None + +def _create_remote_dir(remote_base_path:Path): + """Create the folder that will store pulled documents""" + match remote_base_path.exists(): + case True: log.debug("Skipping creation folder already exists {} in {}.".format(remote_base_path.parent, remote_base_path.parts[-1])) + case False: log.info("Creating folder {} in {}".format(remote_base_path.parts[-1], remote_base_path.parent)) + try: + remote_base_path.mkdir(exist_ok=True) + except Exception as e: + tb_str = ''.join(format_exception(None, e, e.__traceback__)) + raise DoorstopError("""Something unexpected has happened when trying to create folder {} in {}\n. + Details: {}""".format(remote_base_path.parent, remote_base_path.parts[-1], tb_str)) + +def _remote_documents_check(existing_remote_docs: Mapping[str,str], new_remote_docs: Mapping[str,str]) -> dict[str,bool]: + results: dict[str,bool] = {} + + for key in new_remote_docs.keys(): + if key in existing_remote_docs: + results[key] = existing_remote_docs[key] == new_remote_docs[key] + + if list(results.values()).count(False) >=1 : + raise DoorstopError(f""" +Different version for a prefix detected please check the remote document definitions. +Affected prefix(es) { [ x for x in results.keys() if results[x] == False ] } + """) + return results + +def _validate_remote_documents(new_remote_doc: Path, + remote_document_tag: str, + remote_documents: dict[str,str], + doorstop_documents: list[Document] + ): + + root = new_remote_doc.parent.parent + + def find_remote_docs(remote_path:Path): + """ + Recreating part of the build here to discover documents inside the downloaded repo. + """ + + remote_docs: list[Document] = [] + skip_file_name = ".doorstop.skip-all" + exclude_dirnames = {".git", ".venv", "venv"} + for dirpath, dirnames, _ in os.walk(remote_path, topdown=True): + for dirname in dirnames: + if dirname in exclude_dirnames: + continue + path = os.path.join(dirpath, dirname) + if os.path.isfile(os.path.join(path, skip_file_name)): + continue + _document_from_path(path, str(root), remote_docs, remote_documents, new_remote_doc) + return remote_docs + + new_remote_docs = find_remote_docs(new_remote_doc) + entries = { str(d.prefix): remote_document_tag for d in new_remote_docs} + document_check = _remote_documents_check(remote_documents, entries) + remote_documents.update(entries) + data=[ e for e in new_remote_docs if str(e.prefix) not in document_check.keys()] + doorstop_documents.extend(data) + + + +def _git_pull(git_url:str, git_tag:str, target_folder:Path): + + def exact_want(refs, depth=None): + tag = b'refs/heads/' + git_tag.encode() if git_tag in ['main', 'master'] else b'refs/tags/' + git_tag.encode() + if tag in refs: + return tag + + raise DoorstopError( + "ref {} not found in remote {}".format(git_tag, git_url) + ) + + path = str(target_folder.absolute()) + with open(os.devnull, "wb") as f: + if not (target_folder / ".git").exists(): + Repo.init(path, mkdir=False) + porcelain.pull(path, git_url, refspecs=exact_want(porcelain.fetch(path, git_url)), outstream=f,force=True) + + + + +def _download_remote_parent(remote_base_path:Path, + remote_parent_url:str, + remote_parent_tag: str, + doorstop_documents: list[Document], + remote_documents: dict[str,str]): + + _create_remote_dir(remote_base_path) + + folder_name = remote_parent_url.split("/")[-1].split(".")[0] + + if not folder_name: + raise DoorstopError( + "Couldn't guess a name to storage remote parent, check the config for {}" + .format(remote_parent_url)) + + try: + target = remote_base_path / folder_name + _create_remote_dir(target) + _git_pull(remote_parent_url, remote_parent_tag, target) + _validate_remote_documents(target , remote_parent_tag,remote_documents,doorstop_documents) + except Exception as e: + tb_str = ''.join(format_exception(None, e, e.__traceback__)) + raise DoorstopError("""Unexpected error when downloading remote parent from {} +ERROR Details: + {} """.format(remote_parent_url, tb_str)) + diff --git a/doorstop/core/document.py b/doorstop/core/document.py index 356842ab2..0e0518d46 100644 --- a/doorstop/core/document.py +++ b/doorstop/core/document.py @@ -77,6 +77,8 @@ def __init__(self, path, root=os.getcwd(), **kwargs): self._items: List[Item] = [] self._itered = False self.children: List[Document] = [] + self.remote_parent_url: str|None = None + self.remote_parent_tag: str|None = None if not self._data["itemformat"]: self._data["itemformat"] = Item.DEFAULT_ITEMFORMAT @@ -236,6 +238,8 @@ def load(self, reload=False): raise DoorstopError(msg) self.extensions = data.get("extensions", {}) + self.remote_parent_url = data.get("remote_parent_url", None) + self.remote_parent_tag = data.get("remote_parent_tag", None) # Set meta attributes self._loaded = True diff --git a/poetry.lock b/poetry.lock index 13823b314..e9d009cba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -399,6 +399,72 @@ files = [ {file = "docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491"}, ] +[[package]] +name = "dulwich" +version = "0.22.1" +description = "Python Git Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dulwich-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:892914dc2e80403d16e59a3b440044f6092fde5ffd4ec1fdf36d6ff20a8e624d"}, + {file = "dulwich-0.22.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b03092399f0f5d3e112405b890128afdb9e1f203eafb812f5d9105b0f5fac9d4"}, + {file = "dulwich-0.22.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a2c790517ed884bc1b1590806222ab44989eeb7e7f14dfa915561c7ee3b9ac73"}, + {file = "dulwich-0.22.1-cp310-cp310-win32.whl", hash = "sha256:524d3497a86f79959c9c1d729715d60d8171ed3f71621d45afb4faee5a47e8a1"}, + {file = "dulwich-0.22.1-cp310-cp310-win_amd64.whl", hash = "sha256:d3146843b972f744aed551e8ac9fac5714baa864393e480586d467b7b4488426"}, + {file = "dulwich-0.22.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:82f26e592e9a36ab33bcdb419c7d53320e26c85dfc254cdb84f5f561a2fcaabf"}, + {file = "dulwich-0.22.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e90b8a2f24149c5803b733a24f1a016a2943b1f5a9ab2360db545e4638354c35"}, + {file = "dulwich-0.22.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b069639757b2f287f9cd0daf6765b536558c5e28263bbbb28e3d1925bce75400"}, + {file = "dulwich-0.22.1-cp311-cp311-win32.whl", hash = "sha256:3ae006498fea11515027a417e6e68b82e1c195d3516188ba2cc08210e3022d14"}, + {file = "dulwich-0.22.1-cp311-cp311-win_amd64.whl", hash = "sha256:a18d1392eabd02f337dcba23d723a4dcca87274ce8693cf88e6320f38bc3fdcd"}, + {file = "dulwich-0.22.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:12482e318895da9acabea7c0cc70b35d36833e7cb2def511ab3a63617f5c1af3"}, + {file = "dulwich-0.22.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dc42afedc8cda4f2fd15a06d2e9e41281074a02cdf31bb2e0dde4d80766a408"}, + {file = "dulwich-0.22.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3c7774232a2c9b195bde4fb72ad71455e877a9e4e9c0b17a57b1d9bd478838c"}, + {file = "dulwich-0.22.1-cp312-cp312-win32.whl", hash = "sha256:41dfc52db29a06fe23a5029abc3bc13503e28233b1c3a9614bc1e5c4d6adc1ce"}, + {file = "dulwich-0.22.1-cp312-cp312-win_amd64.whl", hash = "sha256:9d19f04ecd4628a0e4587b4c4e98e040b87924c1362ae5aa27420435f05d5dd8"}, + {file = "dulwich-0.22.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0d72a88c7af8fafa14c8743e8923c8d46bd0b850a0b7f5e34eb49201f1ead88e"}, + {file = "dulwich-0.22.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e36c8a3bb09db5730b3d5014d087bce977e878825cdd7ba8285abcd81c43bc0"}, + {file = "dulwich-0.22.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd51e77ff1b4ca08bc9b09b85646a3e77f275827b7b30180d76d769ce608e64d"}, + {file = "dulwich-0.22.1-cp37-cp37m-win32.whl", hash = "sha256:739ef91aeb13fa2aa187d0efd46d0ac168301f54a6ef748565c42876b4b3ce71"}, + {file = "dulwich-0.22.1-cp37-cp37m-win_amd64.whl", hash = "sha256:91966b7b48ec939e5083b03c9154fc450508056f01650ecb58724095307427f5"}, + {file = "dulwich-0.22.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:86be1e283d78cc3f7daee1dcd0254122160cde71ca8c5348315156045f8ac2bb"}, + {file = "dulwich-0.22.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926d671654a2f8cfa0b2fcb6b0c46833af95b5265d27a5c56c49c5a10f3ff3ba"}, + {file = "dulwich-0.22.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:467ff87fc4c61a23424b32acd952476ba17e7f7eeb8090e5957f68129784a35f"}, + {file = "dulwich-0.22.1-cp38-cp38-win32.whl", hash = "sha256:f9e10678fe0692c5167553981d97cbe342ed055c49016aef10da336e2962b1f2"}, + {file = "dulwich-0.22.1-cp38-cp38-win_amd64.whl", hash = "sha256:6386165c64ba5f61c416301f7f32bb899f8200ca575d76888697a42fda8a92d2"}, + {file = "dulwich-0.22.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:84db8aef08df6431b017cc3abe57b3d6885fd7436eec8d715603c309353b233c"}, + {file = "dulwich-0.22.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:155124219e6ce4b116d30ed1b9cc63c88021966b29ce761d3ce3caba064f9a13"}, + {file = "dulwich-0.22.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7579429e89deac6659b4ea70eb3de9063bb40508fd4a8a020fa67b02e0b59f4f"}, + {file = "dulwich-0.22.1-cp39-cp39-win32.whl", hash = "sha256:454d073e628043dde4f9bd34517736c1889dbe6417099bbae2119873b8d4d5da"}, + {file = "dulwich-0.22.1-cp39-cp39-win_amd64.whl", hash = "sha256:e1b51d26108a832f151da8856a93676cc1a5cd8dd0bc20f06f4aee5774a7f0f9"}, + {file = "dulwich-0.22.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4c509e8172b9438536946097768413f297229b03eff064e4e06749cf5c28df78"}, + {file = "dulwich-0.22.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64eebe1d709539d6e80440fa1185f1eeb260d53bcb6435b1f753b4ce90a65e82"}, + {file = "dulwich-0.22.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52a8ddd1d9b5681de216a7af718720f5202d3c093ecc10dd4dfac6d25da605a6"}, + {file = "dulwich-0.22.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7244b796dd7e191753b822ac0ca871a4b9139b0b850770ac5bd347d5f8c76768"}, + {file = "dulwich-0.22.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7798e842ec506d25f21e825259b70109325ac1c9b43c2e287aad7559455951b"}, + {file = "dulwich-0.22.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9fcf7c5cf1e9d0bcc643672f81bf43ec81f6495b99809649f5bfcdff633ab0"}, + {file = "dulwich-0.22.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e346c1b86c5e5f175ca8f87f3873eea8b2e0eeb5d52033b475cf85641cb200c2"}, + {file = "dulwich-0.22.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b2054e1f2c7041857ce129443bde23298ca37592ce82f0fb5ed5704d5f3708dd"}, + {file = "dulwich-0.22.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3ad3462d070b678fe61d3bdd7c6ac3fdbd25cca66f32b6edf589dd88fff251d2"}, + {file = "dulwich-0.22.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc6575476539c0b4924abab3896fc76be2f413d5baa6b083c4dfc4abc59329e"}, + {file = "dulwich-0.22.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3de7a2eba26ff13a79670f78f73fc86fb8c87100508119f3b6bd61451bfdd4bf"}, + {file = "dulwich-0.22.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b2c3186cf76d598a9d42b35e09ef35d499118b4197624ba5bba1b3a39ac6a75f"}, + {file = "dulwich-0.22.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3a366cdba24b8df31ad70b82bae55baa696c453678c1346da8390396a1d5cce4"}, + {file = "dulwich-0.22.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fb61891b2675664dc89d486eb5199e3659179ae04fc0a863ffc7e16b782b624"}, + {file = "dulwich-0.22.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ea4c5feedd35e8bde175a9ab91ef6705c3cef5ee209eeb2f67dd0b59ff1825f"}, + {file = "dulwich-0.22.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:20f61f6dc0b075ca6459acbfb9527ee0e1ee02dccbed126dc492600bb7310d86"}, + {file = "dulwich-0.22.1.tar.gz", hash = "sha256:e36d85967cfbf25da1c7bc3d6921adc5baa976969d926aaf1582bd5fd7e94758"}, +] + +[package.dependencies] +setuptools = {version = "*", markers = "python_version >= \"3.12\""} +urllib3 = ">=1.25" + +[package.extras] +fastimport = ["fastimport"] +https = ["urllib3 (>=1.24.1)"] +paramiko = ["paramiko"] +pgp = ["gpg"] + [[package]] name = "et-xmlfile" version = "1.1.0" @@ -1495,4 +1561,4 @@ tests = ["PasteDeploy", "WSGIProxy2", "coverage", "pyquery", "pytest", "pytest-c [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "c6c6426fff11f1ed567d92b07f7ddc42790df30b0d88d2cb4b1b83b2401fa3c5" +content-hash = "ee7cbdb197e9ec82dec5864df0ddcad8c15ef0881b38ec1836b85113f0e0218c" diff --git a/pyproject.toml b/pyproject.toml index 1921e475f..9d7f5383c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ plantuml-markdown = "^3.4.2" six = "*" # fixes https://github.com/dougn/python-plantuml/issues/11 openpyxl = ">=3.1.2" setuptools = { version = ">=70", python = ">=3.12" } +dulwich = "^0.22.1" [tool.poetry.dev-dependencies] diff --git a/reqs/.doorstop.yml b/reqs/.doorstop.yml index c254e807f..2593c4a85 100644 --- a/reqs/.doorstop.yml +++ b/reqs/.doorstop.yml @@ -1,6 +1,9 @@ +remote_parent_url: git@gitlab.codethink.co.uk:sif/process/trustable.git +remote_parent_tag: main settings: digits: 3 prefix: REQ + parent: T sep: '' attributes: defaults: