Skip to content

Commit

Permalink
WIP make Jupytext's contents manager async
Browse files Browse the repository at this point in the history
  • Loading branch information
@mwouts
mwouts committed Dec 10, 2022
1 parent 3642829 commit 45fde60
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 96 deletions.
136 changes: 83 additions & 53 deletions jupytext/contentsmanager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from datetime import timedelta

import nbformat
from jupyter_core.utils import ensure_async
from tornado.web import HTTPError

# import notebook.transutils before notebook.services.contents.filemanager #75
Expand Down Expand Up @@ -101,28 +102,30 @@ def update_paired_notebooks(self, path, formats):
short_formats,
)

def create_prefix_dir(self, path, fmt):
async def create_prefix_dir(self, path, fmt):
"""Create the prefix dir, if missing"""
if "prefix" in fmt and "/" in path:
parent_dir = self.get_parent_dir(path)
if not self.dir_exists(parent_dir):
self.create_prefix_dir(parent_dir, fmt)
self.log.info("Creating directory %s", parent_dir)
self.super.save(dict(type="directory"), parent_dir)
await ensure_async(
self.super.save(dict(type="directory"), parent_dir)
)

def save(self, model, path=""):
async def save(self, model, path=""):
"""Save the file model and return the model with no content."""
if model["type"] != "notebook":
return self.super.save(model, path)
await ensure_async(self.super.save(model, path))

path = path.strip("/")
nbk = model["content"]
try:
config = self.get_config(path)
config = await self.get_config(path)
jupytext_formats = notebook_formats(nbk, config, path)
self.update_paired_notebooks(path, jupytext_formats)

def save_one_file(path, fmt):
async def save_one_file(path, fmt):
if "format_name" in fmt and fmt["extension"] not in [
".md",
".markdown",
Expand All @@ -139,14 +142,16 @@ def save_one_file(path, fmt):

self.create_prefix_dir(path, fmt)
if fmt["extension"] == ".ipynb":
return self.super.save(
dict(
type="notebook",
content=drop_text_representation_metadata(
model["content"]
return await ensure_async(
self.super.save(
dict(
type="notebook",
content=drop_text_representation_metadata(
model["content"]
),
),
),
path,
path,
)
)

if (
Expand All @@ -170,15 +175,15 @@ def save_one_file(path, fmt):
),
)

return self.super.save(text_model, path)
return await ensure_async(self.super.save(text_model, path))

return write_pair(path, jupytext_formats, save_one_file)
return await write_pair(path, jupytext_formats, save_one_file)

except Exception as e:
self.log.error("Error while saving file: %s %s", path, e, exc_info=True)
raise HTTPError(500, f"Unexpected error while saving file: {path} {e}")

def get(
async def get(
self,
path,
content=True,
Expand All @@ -192,21 +197,25 @@ def get(

# Not a notebook?
if (
not self.file_exists(path)
or self.dir_exists(path)
not await self.file_exists(path)
or await self.dir_exists(path)
or (type is not None and type != "notebook")
):
return self.super.get(path, content, type, format)
return await ensure_async(self.super.get(path, content, type, format))

config = self.get_config(path, use_cache=content is False)
config = await self.get_config(path, use_cache=content is False)
if ext not in self.all_nb_extensions(config):
return self.super.get(path, content, type, format)
return await ensure_async(self.super.get(path, content, type, format))

fmt = preferred_format(ext, config.preferred_jupytext_formats_read)
if ext == ".ipynb":
model = self.super.get(path, content, type="notebook", format=format)
model = await ensure_async(
self.super.get(path, content, type="notebook", format=format)
)
else:
model = self.super.get(path, content, type="file", format="text")
model = await ensure_async(
self.super.get(path, content, type="file", format="text")
)
model["type"] = "notebook"
if content:
# We may need to update these keys, inherited from text files formats
Expand Down Expand Up @@ -275,29 +284,39 @@ def get(
if jupytext_metadata:
model["content"]["metadata"]["jupytext"] = jupytext_metadata

def get_timestamp(alt_path):
async def get_timestamp(alt_path):
if not self.exists(alt_path):
return None
if alt_path == path:
return model["last_modified"]
return self.super.get(alt_path, content=False)["last_modified"]
return (await ensure_async(self.super.get(alt_path, content=False)))[
"last_modified"
]

def read_one_file(alt_path, alt_fmt):
async def read_one_file(alt_path, alt_fmt):
if alt_path == path:
return model["content"]
if alt_path.endswith(".ipynb"):
self.log.info(f"Reading OUTPUTS from {alt_path}")
return self.super.get(
alt_path, content=True, type="notebook", format=format
return (
await ensure_async(
self.super.get(
alt_path, content=True, type="notebook", format=format
)
)
)["content"]

self.log.info(f"Reading SOURCE from {alt_path}")
text = self.super.get(
alt_path, content=True, type="file", format=format
text = (
await ensure_async(
self.super.get(
alt_path, content=True, type="file", format=format
)
)
)["content"]
return reads(text, fmt=alt_fmt, config=config)

inputs, outputs = latest_inputs_and_outputs(
inputs, outputs = await latest_inputs_and_outputs(
path, fmt, formats, get_timestamp, contents_manager_mode=True
)

Expand Down Expand Up @@ -329,7 +348,7 @@ def read_one_file(alt_path, alt_fmt):
self.log.warning(ts_mismatch)

try:
content = read_pair(
content = await read_pair(
inputs, outputs, read_one_file, must_match=True
)
self.log.warning(
Expand Down Expand Up @@ -367,7 +386,7 @@ def read_one_file(alt_path, alt_fmt):
model["content"] = content
else:
try:
model["content"] = read_pair(inputs, outputs, read_one_file)
model["content"] = await read_pair(inputs, outputs, read_one_file)
except HTTPError:
raise
except Exception as err:
Expand All @@ -381,14 +400,14 @@ def read_one_file(alt_path, alt_fmt):

return model

def new_untitled(self, path="", type="", ext=""):
async def new_untitled(self, path="", type="", ext=""):
"""Create a new untitled file or directory in path
We override the base function because that one does not take the 'ext' argument
into account when type=="notebook". See https://github.com/mwouts/jupytext/issues/443
"""
if type != "notebook" and ext != ".ipynb":
return self.super.new_untitled(path, type, ext)
return await ensure_async(self.super.new_untitled(path, type, ext))

ext = ext or ".ipynb"
if ":" in ext:
Expand All @@ -401,7 +420,7 @@ def new_untitled(self, path="", type="", ext=""):
raise HTTPError(404, "No such directory: %s" % path)

untitled = self.untitled_notebook
config = self.get_config(path)
config = await self.get_config(path)
name = self.increment_notebook_filename(config, untitled + ext, path)
path = f"{path}/{name}"

Expand Down Expand Up @@ -435,18 +454,18 @@ def increment_notebook_filename(self, config, filename, path=""):
break
return name

def trust_notebook(self, path):
async def trust_notebook(self, path):
"""Trust the current notebook"""
if path.endswith(".ipynb") or path not in self.paired_notebooks:
self.super.trust_notebook(path)
await ensure_async(self.super.trust_notebook(path))
return

fmt, formats = self.paired_notebooks[path]
for alt_path, alt_fmt in paired_paths(path, fmt, formats):
if alt_fmt["extension"] == ".ipynb":
self.super.trust_notebook(alt_path)
await ensure_async(self.super.trust_notebook(alt_path))

def rename_file(self, old_path, new_path):
async def rename_file(self, old_path, new_path):
"""Rename the current notebook, as well as its alternative representations"""
if old_path not in self.paired_notebooks:
try:
Expand All @@ -457,7 +476,7 @@ def rename_file(self, old_path, new_path):
pass

if old_path not in self.paired_notebooks:
self.super.rename_file(old_path, new_path)
await ensure_async(self.super.rename_file(old_path, new_path))
return

fmt, formats = self.paired_notebooks.get(old_path)
Expand All @@ -481,7 +500,9 @@ def rename_file(self, old_path, new_path):
for old_alt_path, alt_fmt in old_alt_paths:
new_alt_path = full_path(new_base, alt_fmt)
if self.exists(old_alt_path):
self.super.rename_file(old_alt_path, new_alt_path)
await ensure_async(
self.super.rename_file(old_alt_path, new_alt_path)
)

self.drop_paired_notebook(old_path)
self.update_paired_notebooks(new_path, formats)
Expand All @@ -497,11 +518,11 @@ def get_parent_dir(self, path):
return path.rsplit(":", 1)[0] + ":"
return ""

def get_config_file(self, directory):
async def get_config_file(self, directory):
"""Return the jupytext configuration file, if any"""
for jupytext_config_file in JUPYTEXT_CONFIG_FILES:
path = directory + "/" + jupytext_config_file
if self.file_exists(path):
if await self.file_exists(path):
if not self.allow_hidden and jupytext_config_file.startswith("."):
self.log.warning(
f"Ignoring config file {path} (see Jupytext issue #964)"
Expand All @@ -510,10 +531,10 @@ def get_config_file(self, directory):
return path

pyproject_path = directory + "/" + PYPROJECT_FILE
if self.file_exists(pyproject_path):
if await self.file_exists(pyproject_path):
import toml

model = self.get(pyproject_path, type="file")
model = await self.get(pyproject_path, type="file")
doc = toml.loads(model["content"])
if doc.get("tool", {}).get("jupytext") is not None:
return pyproject_path
Expand All @@ -522,9 +543,9 @@ def get_config_file(self, directory):
return None

parent_dir = self.get_parent_dir(directory)
return self.get_config_file(parent_dir)
return await self.get_config_file(parent_dir)

def load_config_file(
async def load_config_file(
self, config_file, *, prev_config_file, prev_config, is_os_path=False
):
"""Load the configuration file"""
Expand All @@ -537,7 +558,9 @@ def load_config_file(
config_content = None
if not is_os_path:
try:
model = self.super.get(config_file, content=True, type="file")
model = await ensure_async(
self.super.get(config_file, content=True, type="file")
)
config_content = model["content"]
except HTTPError:
pass
Expand All @@ -558,7 +581,7 @@ def load_config_file(
)
return config

def get_config(self, path, use_cache=False):
async def get_config(self, path, use_cache=False):
"""Return the Jupytext configuration for the given path"""
parent_dir = self.get_parent_dir(path)

Expand All @@ -567,16 +590,16 @@ def get_config(self, path, use_cache=False):
# to a different directory.
if not use_cache or parent_dir != self.cached_config.path:
try:
config_file = self.get_config_file(parent_dir)
config_file = await self.get_config_file(parent_dir)
if config_file:
self.cached_config.config = self.load_config_file(
self.cached_config.config = await self.load_config_file(
config_file,
prev_config_file=self.cached_config.config_file,
prev_config=self.cached_config.config,
)
else:
config_file = find_global_jupytext_configuration_file()
self.cached_config.config = self.load_config_file(
self.cached_config.config = await self.load_config_file(
config_file,
prev_config_file=self.cached_config.config_file,
prev_config=self.cached_config.config,
Expand All @@ -603,6 +626,13 @@ def get_config(self, path, use_cache=False):


try:
# If possible, we derive Jupytext's contents manager from the default one
from jupyter_server.serverapp import ServerApp

TextFileContentsManager = build_jupytext_contents_manager_class(
ServerApp.contents_manager_class.default()
)
except ImportError:
# The LargeFileManager is taken by default from jupyter_server if available
from jupyter_server.services.contents.largefilemanager import LargeFileManager

Expand Down
Loading

0 comments on commit 45fde60

Please sign in to comment.