Implement formatters for ls operation

python/CHANGELOG.D/427.feature

@@ -0,0 +1 @@
+Changes in `neuro store ls` behavior: display files by columns by default, add option `-l` for long output, display one per line for pipes by default.

python/README.md

@@ -75,7 +75,7 @@ Name | Description|
 |----|------------|
 |_\-v, --verbose_|Enable verbose mode|
 |_\--show-traceback_|Show python traceback on error, useful for debugging the tool.|
-|_--color \[yes|no|auto]_|Color mode|
+|_--color \[yes | no | auto]_|Color mode|
 |_--version_|Show the version and exit.|
 |_--help_|Show this message and exit.|
 
@@ -262,7 +262,7 @@ neuro job list -s pending -s running -q
 
 Name | Description|
 |----|------------|
-|_\-s, --status \[pending|running|succeeded|failed|all]_|Filter out job by status \(multiple option)|
+|_\-s, --status \[pending | running | succeeded | failed | all]_|Filter out job by status \(multiple option)|
 |_\-d, --description DESCRIPTION_|Filter out job by job description \(exact match)|
 |_\-q, --quiet_||
 |_--help_|Show this message and exit.|
@@ -455,6 +455,9 @@ neuro storage ls [OPTIONS] [PATH]
 
 Name | Description|
 |----|------------|
+|_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M)|
+|_-l_|use a long listing format|
+|_--sort \[name | size | time]_|sort by given field, default is name|
 |_--help_|Show this message and exit.|
 
 
@@ -814,7 +817,7 @@ neuro completion generate [OPTIONS]
 
 Name | Description|
 |----|------------|
-|_--shell \[bash|zsh]_|Shell type. \[default: bash]|
+|_--shell \[bash | zsh]_|Shell type. \[default: bash]|
 |_--help_|Show this message and exit.|
 
 
@@ -834,7 +837,7 @@ neuro completion patch [OPTIONS]
 
 Name | Description|
 |----|------------|
-|_--shell \[bash|zsh]_|Shell type. \[default: bash]|
+|_--shell \[bash | zsh]_|Shell type. \[default: bash]|
 |_--help_|Show this message and exit.|
 
 
@@ -913,7 +916,7 @@ neuro job list -s pending -s running -q
 
 Name | Description|
 |----|------------|
-|_\-s, --status \[pending|running|succeeded|failed|all]_|Filter out job by status \(multiple option)|
+|_\-s, --status \[pending | running | succeeded | failed | all]_|Filter out job by status \(multiple option)|
 |_\-d, --description DESCRIPTION_|Filter out job by job description \(exact match)|
 |_\-q, --quiet_||
 |_--help_|Show this message and exit.|
@@ -1105,6 +1108,9 @@ neuro ls [OPTIONS] [PATH]
 
 Name | Description|
 |----|------------|
+|_\-h, --human-readable_|with -l print human readable sizes \(e.g., 2K, 540M)|
+|_-l_|use a long listing format|
+|_--sort \[name | size | time]_|sort by given field, default is name|
 |_--help_|Show this message and exit.|
 
 

python/build-tools/cli-help-generator.py

@@ -107,6 +107,11 @@ def escape(text: str) -> str:
  escaped.append(after)
  return "<br/>".join(escaped)
 
+ def escape_cell(text: str) -> str:
+ escaped = escape(text)
+ escaped = re.sub(r"\|", r"&#124;", escaped)
+ return escaped
+
  md = ""
  md += f"{header_prefix}# {info.name}"
  md += "\n\n"
@@ -132,7 +137,11 @@ def escape(text: str) -> str:
  md += "Name | Description|\n"
  md += "|----|------------|\n"
  for option in info.options:
- md += f"|_{escape(option.pattern)}_|{escape(option.description)}|\n"
+ md += (
+ f"|_{escape_cell(option.pattern.replace('|', ' | '))}_"
+ f"|{escape_cell(option.description)}|"
+ f"\n"
+ )
 
  md += "\n\n"
 

python/neuromation/cli/files_formatter.py

@@ -0,0 +1,134 @@
+import abc
+import enum
+import time
+from math import ceil
+from typing import Any, Iterator, List, Sequence
+
+import humanize
+
+from neuromation.cli.formatter import BaseFormatter
+from neuromation.client.storage import FileStatus, FileStatusType
+
+
+RECENT_TIME_DELTA = 365 * 24 * 60 * 60 / 2
+TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+def chunks(list: Sequence[Any], size: int) -> Sequence[Any]:
+ result = []
+ for i in range(0, len(list), size):
+ result.append(list[i : i + size])
+ return result
+
+
+def transpose(columns: Sequence[Sequence[Any]]) -> Sequence[Sequence[Any]]:
+ height = len(columns)
+ width = len(columns[0])
+ result: Sequence[List[Any]] = [[] for _ in range(width)]
+ for i in range(width):
+ for j in range(height):
+ if i < len(columns[j]):
+ result[i].append(columns[j][i])
+ return result
+
+
+class BaseFilesFormatter(BaseFormatter, abc.ABC):
+ def format(self, files: Sequence[FileStatus]) -> Iterator[str]: # pragma: no cover
+ pass
+
+
+class LongFilesFormatter(BaseFilesFormatter):
+ def __init__(self, human_readable: bool = False):
+ self.human_readable = human_readable
+
+ def _columns_for_file(self, file: FileStatus) -> Sequence[str]:
+ type = "-"
+ if file.type == FileStatusType.DIRECTORY:
+ type = "d"
+
+ permission = file.permission[0]
+
+ date = time.strftime(TIME_FORMAT, time.localtime(file.modification_time))
+
+ size = file.size
+ if self.human_readable:
+ size = humanize.naturalsize(size, gnu=True).rstrip("B")
+
+ name = file.name
+
+ return [f"{type}{permission}", f"{size}", f"{date}", f"{name}"]
+
+ def format(self, files: Sequence[FileStatus]) -> Iterator[str]:
+ if not files:
+ return
+ table = [self._columns_for_file(file) for file in files]
+ widths = [0 for _ in table[0]]
+ for row in table:
+ for x in range(len(row)):
+ if widths[x] < len(row[x]):
+ widths[x] = len(row[x])
+ for row in table:
+ line = []
+ for x in range(len(row)):
+ if x == len(row) - 1:
+ line.append(row[x])
+ else:
+ line.append(row[x].rjust(widths[x]))
+ yield " ".join(line)
+
+
+class SimpleFilesFormatter(BaseFilesFormatter):
+ def format(self, files: Sequence[FileStatus]) -> Iterator[str]:
+ for file in files:
+ yield f"{file.name}"
+
+
+class VerticalColumnsFilesFormatter(BaseFilesFormatter):
+ def __init__(self, width: int):
+ self.width = width
+
+ def format(self, files: Sequence[FileStatus]) -> Iterator[str]:
+ if not files:
+ return
+ items = [f"{file.name}" for file in files]
+ widths = [len(item) for item in items]
+ # let`s check how many columns we can use
+ test_count = 1
+ while True:
+ test_columns = chunks(widths, ceil(len(items) / test_count))
+ test_columns_widths = [max(column) for column in test_columns]
+ test_total_width = sum(test_columns_widths) + 2 * (len(test_columns) - 1)
+ if test_count == 1 or test_total_width <= self.width:
+ count = test_count
+ columns_widths = test_columns_widths
+ if test_total_width == self.width:
+ break
+
+ if test_total_width >= self.width or len(test_columns) == len(items):
+ break
+ test_count = test_count + 1
+
+ rows = transpose(chunks(items, ceil(len(items) / count)))
+ for row in rows:
+ formatted_row = []
+ for i in range(len(row)):
+ formatted = row[i]
+ if i < len(row) - 1:
+ formatted = formatted.ljust(columns_widths[i])
+ formatted_row.append(formatted)
+ yield " ".join(formatted_row)
+
+
+class FilesSorter(str, enum.Enum):
+ NAME = "name"
+ SIZE = "size"
+ TIME = "time"
+
+ def sort(self, files: List[FileStatus]) -> None:
+ if self == self.NAME:
+ field = "name"
+ elif self == self.SIZE:
+ field = "size"
+ elif self == self.TIME:
+ field = "modification_time"
+ files.sort(key=lambda x: x.__getattribute__(field))

python/neuromation/cli/formatter.py

@@ -1,12 +1,12 @@
 import itertools
 import re
 import time
-from typing import AbstractSet, Iterable, List, Optional
+from typing import AbstractSet, Iterable, Optional
 
 import click
 from dateutil.parser import isoparse # type: ignore
 
-from neuromation.client import FileStatus, JobDescription, JobStatus, Resources
+from neuromation.client import JobDescription, JobStatus, Resources
 from neuromation.client.jobs import JobTelemetry
 
 from .rc import Config
@@ -105,16 +105,6 @@ def __call__(self, job: JobDescription, *, finish: bool = False) -> str:
  return ret
 
 
-class StorageLsFormatter(BaseFormatter):
- FORMAT = "{type:<15}{size:<15,}{name:<}".format
-
- def __call__(self, lst: List[FileStatus]) -> str:
- return "\n".join(
- self.FORMAT(type=status.type.lower(), name=status.path, size=status.size)
- for status in lst
- )
-
-
 class JobStatusFormatter(BaseFormatter):
  def __call__(self, job_status: JobDescription) -> str:
  result: str = f"Job: {job_status.id}\n"

python/neuromation/cli/storage.py

@@ -1,16 +1,24 @@
 import logging
+import shutil
+import sys
 
 import aiohttp
 import click
 from yarl import URL
 
+from neuromation.cli.files_formatter import (
+ BaseFilesFormatter,
+ FilesSorter,
+ LongFilesFormatter,
+ SimpleFilesFormatter,
+ VerticalColumnsFilesFormatter,
+)
 from neuromation.client.url_utils import (
  normalize_local_path_uri,
  normalize_storage_path_uri,
 )
 
 from .command_progress_report import ProgressBase
-from .formatter import StorageLsFormatter
 from .rc import Config
 from .utils import command, group, run_async
 
@@ -48,21 +56,58 @@ async def rm(cfg: Config, path: str) -> None:
 
 @command()
 @click.argument("path", default="storage://~")
+@click.option(
+ "--human-readable",
+ "-h",
+ is_flag=True,
+ help="with -l print human readable sizes (e.g., 2K, 540M)",
+)
+@click.option("-l", "format_long", is_flag=True, help="use a long listing format")
+@click.option(
+ "--sort",
+ type=click.Choice(["name", "size", "time"]),
+ default="name",
+ help="sort by given field, default is name",
+)
 @click.pass_obj
 @run_async
-async def ls(cfg: Config, path: str) -> None:
+async def ls(
+ cfg: Config, path: str, human_readable: bool, format_long: bool, sort: str
+) -> None:
  """
  List directory contents.
 
  By default PATH is equal user`s home dir (storage:)
  """
+ if format_long:
+ formatter: BaseFilesFormatter = LongFilesFormatter(
+ human_readable=human_readable
+ )
+ else:
+ is_tty = sys.stdout.isatty()
+ if is_tty:
+ width, _ = shutil.get_terminal_size((80, 25))
+ formatter = VerticalColumnsFilesFormatter(width=width)
+ else:
+ formatter = SimpleFilesFormatter()
+
+ if sort == "size":
+ sorter = FilesSorter.SIZE
+ elif sort == "time":
+ sorter = FilesSorter.TIME
+ else:
+ sorter = FilesSorter.NAME
+
  uri = normalize_storage_path_uri(URL(path), cfg.username)
  log.info(f"Using path '{uri}'")
 
  async with cfg.make_client() as client:
- res = await client.storage.ls(uri)
+ files = await client.storage.ls(uri)
+
+ sorter.sort(files)
 
- click.echo(StorageLsFormatter()(res))
+ for line in formatter.format(files):
+ click.echo(line)
 
 
 @command()

python/requirements/base.txt

@@ -8,5 +8,6 @@ yarl==1.3.0
 aiodocker>=0.14.0
 click==7.0
 colorama==0.4.1
+humanize==0.5.1
 
 -e .

python/setup.cfg

@@ -1,7 +1,7 @@
 [flake8]
 exclude = .git,.env,__pycache__,.eggs
 max-line-length = 88
-ignore = N801,N802,N803,E252,W503,E133
+ignore = N801,N802,N803,E252,W503,E133,E203
 
 [isort]
 line_length=88
@@ -49,4 +49,7 @@ ignore_missing_imports = true
 ignore_missing_imports = true
 
 [mypy-jose]
+ignore_missing_imports = true
+
+[mypy-humanize]
 ignore_missing_imports = true

python/setup.py

@@ -31,6 +31,7 @@
  "aiodocker>=0.14.0",
  "click>=7.0",
  "colorama>=0.4",
+ "humanize==0.5.1"
  ],
  include_package_data=True,
  description="Neuromation Platform API client",