Add repo owner to doc source links (#334)

* ✨ add repo owner to source links

* 🐛 add version tag suffix

* ✨ add repo_owner flag to build cmd

* ✨ add repo owner to build cmd in workflow

* 🚧 wip

* 🐛 bufix (wip)

* 🚧 wip

* 🐛 add missing version_tag_suffix flag

* 🚧 wip

* 🚧 wip

* 🚧 wip

* Apply suggestions from code review

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

* ✅ make tests pass

* 🐛 use package_name for hf docs url not repo_name

* 📝 some docs and update main github action

* 📝 update comment for repo_name

* 🚧 can I remove git pull?

* 🐛 revert

* ✨ add version tag suffix

* 🐛 use = to allow for empty string in argparse

* ✨ add version tag suffix to main documentation

* 💚 update CI to use main branch before merging

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>

.github/workflows/build_main_documentation.yml

@@ -9,9 +9,10 @@ on:
       package:
         required: true
         type: string
+        description: "Name of the GitHub repo."
       package_name:
         type: string
-        description: "Should be used when a package name differs from its repostory name"
+        description: "Name of the Python package if it differs from repo name."
       path_to_docs:
         type: string
       notebook_folder:
@@ -32,6 +33,10 @@ on:
         type: string
         default: 'huggingface'
         description: "Owner of the repo to build documentation for. Defaults to 'huggingface'."
+      version_tag_suffix:
+        type: string
+        default: "src/"
+        description: "Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links."
     secrets:
       token:
         required: false
@@ -130,7 +135,7 @@ jobs:
         run: |
           echo "doc_folder has been set to ${{ env.doc_folder }}"
           cd doc-builder
-          args="--build_dir ../build_dir --clean --html ${{ inputs.additional_args }}"
+          args="--build_dir ../build_dir --clean --html ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}"
 
           if [ ! -z "${{ inputs.notebook_folder }}" ];
           then

.github/workflows/build_pr_documentation.yml

@@ -35,6 +35,10 @@ on:
         type: string
         default: 'huggingface'
         description: "Owner of the repo to build documentation for. Defaults to 'huggingface'."
+      version_tag_suffix:
+        type: string
+        default: "src/"
+        description: "Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links."
     secrets:
       token:
         required: false
@@ -134,7 +138,7 @@ jobs:
         run: |
           echo "doc_folder has been set to ${{ env.doc_folder }}"
           cd doc-builder
-          args="--build_dir ../build_dir --clean --version pr_${{ inputs.pr_number }} --html ${{ inputs.additional_args }}"
+          args="--build_dir ../build_dir --clean --version pr_${{ inputs.pr_number }} --html ${{ inputs.additional_args }} --repo_owner ${{ inputs.repo_owner }} --repo_name ${{ inputs.package }} --version_tag_suffix=${{ inputs.version_tag_suffix }}"
 
           if [ -z "${{ inputs.languages }}" ];
           then

setup.py

@@ -8,7 +8,7 @@
 extras = {}
 
 extras["transformers"] = ["transformers[dev]"]
-extras["testing"] = ["pytest", "pytest-xdist", "torch", "transformers", "tokenizers"]
+extras["testing"] = ["pytest", "pytest-xdist", "torch", "transformers", "tokenizers", "timm"]
 extras["quality"] = ["black~=22.0", "isort>=5.5.4", "flake8>=3.8.3"]
 
 extras["all"] = extras["testing"] + extras["quality"]

src/doc_builder/autodoc.py

@@ -329,9 +329,11 @@ def get_source_link(obj, page_info, version_tag_suffix="src/"):
     """
     Returns the link to the source code of an object on GitHub.
     """
-    package_name = page_info["package_name"]
+    # Repo name defaults to package_name, but if provided in page_info, it will be used instead.
+    repo_name = page_info.get("repo_name", page_info.get("package_name"))
     version_tag = page_info.get("version_tag", "main")
-    base_link = f"https://github.com/huggingface/{package_name}/blob/{version_tag}/{version_tag_suffix}"
+    repo_owner = page_info.get("repo_owner", "huggingface")
+    base_link = f"https://github.com/{repo_owner}/{repo_name}/blob/{version_tag}/{version_tag_suffix}"
     module = obj.__module__.replace(".", "/")
     line_number = inspect.getsourcelines(obj)[1]
     source_file = inspect.getsourcefile(obj)
@@ -364,8 +366,8 @@ def document_object(object_name, package, page_info, full_name=True, anchor_name
         anchor_name (`str`, *optional*): The name to give to the anchor for this object.
         version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
             Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
-            For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
-            For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+            For example, the default `"src/"` suffix will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/src/`.
+            For example, `version_tag_suffix=""` will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/`.
     """
     if page_info is None:
         page_info = {}
@@ -460,8 +462,8 @@ def autodoc(object_name, package, methods=None, return_anchors=False, page_info=
         page_info (`Dict[str, str]`, *optional*): Some information about the page.
         version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
             Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
-            For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
-            For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+            For example, the default `"src/"` suffix will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/src/`.
+            For example, `version_tag_suffix=""` will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/`.
     """
     if page_info is None:
         page_info = {}

src/doc_builder/build_doc.py

@@ -387,6 +387,8 @@ def build_doc(
     is_python_module=False,
     watch_mode=False,
     version_tag_suffix="src/",
+    repo_owner="huggingface",
+    repo_name=None,
 ):
     """
     Build the documentation of a package.
@@ -412,8 +414,19 @@ def build_doc(
             Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
             For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
             For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+        repo_owner (`str`, *optional*, defaults to `"huggingface"`):
+            The owner of the repository on GitHub. In most cases, this is `"huggingface"`. However, for the `timm` library, the owner is `"rwightman"`.
+        repo_name (`str`, *optional*, defaults to `package_name`):
+            The name of the repository on GitHub. In most cases, this is the same as `package_name`. However, for the `timm` library, the name is `"pytorch-image-models"` instead of `"timm"`.
     """
-    page_info = {"version": version, "version_tag": version_tag, "language": language, "package_name": package_name}
+    page_info = {
+        "version": version,
+        "version_tag": version_tag,
+        "language": language,
+        "package_name": package_name,
+        "repo_owner": repo_owner,
+        "repo_name": repo_name if repo_name is not None else package_name,
+    }
     if clean and Path(output_dir).exists():
         shutil.rmtree(output_dir)
 

src/doc_builder/commands/build.py

@@ -104,6 +104,8 @@ def build_command(args):
         notebook_dir=notebook_dir,
         is_python_module=not args.not_python_module,
         version_tag_suffix=args.version_tag_suffix,
+        repo_owner=args.repo_owner,
+        repo_name=args.repo_name,
     )
 
     # dev build should not update _versions.yml
@@ -203,7 +205,18 @@ def build_command_parser(subparsers=None):
         default="src/",
         help="Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links. For example, the default `src/` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.",
     )
-
+    parser.add_argument(
+        "--repo_owner",
+        type=str,
+        default="huggingface",
+        help="Owner of the repo (e.g. huggingface, rwightman, etc.).",
+    )
+    parser.add_argument(
+        "--repo_name",
+        type=str,
+        default=None,
+        help="Name of the repo (e.g. transformers, pytorch-image-models, etc.). By default, this is the same as the library_name.",
+    )
     if subparsers is not None:
         parser.set_defaults(func=build_command)
     return parser

src/doc_builder/external.py

@@ -54,14 +54,18 @@ def post_process_objects_inv(object_data, doc_url):
     return links
 
 
-def get_stable_version(package_name):
+def get_stable_version(package_name, repo_owner="huggingface", repo_name=None):
     """
     Gets the version of the last release of a package.
 
     Args:
         package_name (`str`): The name of the package.
+        repo_owner (`str`): The owner of the GitHub repo.
+        repo_name (`str`, *optional*, defaults to `package_name`):
+            The name of the GitHub repo. If not provided, will be the same as the package name.
     """
-    github_url = f"https://github.com/huggingface/{package_name}"
+    repo_name = repo_name if repo_name is not None else package_name
+    github_url = f"https://github.com/{repo_owner}/{repo_name}"
     try:
         # Get the version tags from the GitHub repo in decreasing order (that's what '-v:refname' means)
         result = git.cmd.Git().ls_remote(github_url, sort="-v:refname", tags=True)
@@ -80,20 +84,24 @@ def get_stable_version(package_name):
     return "main"
 
 
-def get_objects_map(package_name, version="main", language="en"):
+def get_objects_map(package_name, version="main", language="en", repo_owner="huggingface", repo_name=None):
     """
     Downloads the `objects.inv` for a package and post-processes it to get a nice dictionary.
 
     Args:
         package_name (`str`): The name of the external package.
         version (`str`, *optional*, defaults to `"main"`): The version of the package for which documentation is built.
         language (`str`, *optional*, defaults to `"en"`): The langauge of the documentation being built.
+        repo_owner (`str`, *optional*, defaults to `"huggingface"`): The owner of the GitHub repo.
+        repo_name (`str`, *optional*, defaults to `package_name`):
+            The name of the GitHub repo. If not provided, it will be the same as the package name.
     """
+    repo_name = repo_name if repo_name is not None else package_name
     # We link to main in `package_name` from the main doc (or PR docs) but to the last stable release otherwise.
     if version in ["main", "master"] or version.startswith("pr_"):
         package_version = "main"
     else:
-        package_version = get_stable_version(package_name)
+        package_version = get_stable_version(package_name, repo_owner, repo_name)
 
     doc_url = f"{HF_DOC_PREFIX}{package_name}/{package_version}/{language}"
     url = f"{doc_url}/objects.inv"

tests/test_autodoc.py

@@ -19,6 +19,7 @@
 from dataclasses import dataclass
 from typing import List, Optional, Union
 
+import timm
 import transformers
 from doc_builder.autodoc import (
     autodoc,
@@ -39,9 +40,10 @@
 from transformers.utils import PushToHubMixin
 
 
-# This is dynamic since the Transformers library is not frozen.
+# This is dynamic since the Transformers/timm libraries are not frozen.
 TEST_LINE_NUMBER = inspect.getsourcelines(transformers.utils.ModelOutput)[1]
 TEST_LINE_NUMBER2 = inspect.getsourcelines(transformers.pipeline)[1]
+TEST_LINE_NUMBER_TIMM = inspect.getsourcelines(timm.create_model)[1]
 
 TEST_DOCSTRING = """Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.
 
@@ -140,6 +142,9 @@ class AutodocTester(unittest.TestCase):
         f"https://github.com/huggingface/transformers/blob/main/src/transformers/utils/generic.py#L{TEST_LINE_NUMBER}"
     )
     test_source_link_init = f"https://github.com/huggingface/transformers/blob/main/src/transformers/pipelines/__init__.py#L{TEST_LINE_NUMBER2}"
+    test_source_link_timm = (
+        f"https://github.com/rwightman/pytorch-image-models/blob/main/timm/models/factory.py#L{TEST_LINE_NUMBER_TIMM}"
+    )
 
     def test_find_object_in_package(self):
         self.assertEqual(find_object_in_package("BertModel", transformers), BertModel)
@@ -248,6 +253,12 @@ def test_get_source_link(self):
         self.assertEqual(get_source_link(transformers.utils.ModelOutput, page_info), self.test_source_link)
         self.assertEqual(get_source_link(transformers.pipeline, page_info), self.test_source_link_init)
 
+    def test_get_source_link_different_repo_owner(self):
+        page_info = {"package_name": "timm", "repo_owner": "rwightman", "repo_name": "pytorch-image-models"}
+        self.assertEqual(
+            get_source_link(timm.create_model, page_info, version_tag_suffix=""), self.test_source_link_timm
+        )
+
     def test_document_object(self):
         page_info = {"package_name": "transformers"}