Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

修复win环境下部分指标无法跟踪的问题 #641

Merged
merged 6 commits into from
Jun 27, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions swanlab/data/formater.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import re
import json
import yaml
from typing import List


def check_string(target: str) -> bool:
Expand Down Expand Up @@ -159,5 +160,69 @@ def check_key_format(key: str, auto_cut=True) -> str:
raise ValueError(f"tag: {key} is an empty string")
if key.startswith((".", "/")):
raise ValueError(f"tag: {key} can't start with '.' or '/' and blank space")
if key.endswith((".", "/")): # cannot create folder end with '.' or '/'
raise ValueError(f"tag: {key} can't end with '.' or '/' and blank space")
# 检查长度
return _auto_cut("tag", key, max_len, auto_cut)


def check_unique_on_case_insensitive(names: List[str]):
ShaohonChen marked this conversation as resolved.
Show resolved Hide resolved
"""
Ensure that the names are unique in case-insensitive.

Parameters
----------
names : List[str]
List of names.

Returns
-------
bool
True if names are unique

Raises
------
ValueError
names are not unique
"""
exist_names_set = set()
for n in names:
n_lower = n.lower()
if n_lower in exist_names_set:
raise ValueError(f'tag: Windows is case insensitive, find same name: "{n}"')
exist_names_set.add(n_lower)

return True


def check_win_reserved_folder_name(folder_name: str, auto_fix=True) -> str:
Zeyi-Lin marked this conversation as resolved.
Show resolved Hide resolved
"""
Check if a folder name is reserved or not support to Windows.

Parameters
----------
folder_name : str
Name of the folder to check.
auto_fix : bool, optional
auto fix unsupport folder_name, default True
If the value is False, try to throw an ValueError

Returns
-------
bool
return fix name

Raises
------
ValueError
key not support to Windows.
"""
# Regular expression to match reserved names optionally followed by a dot (.) and any character
reserved_pattern = re.compile(r"^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\..*)?$", re.IGNORECASE)

# Check if the cleaned folder name is in the reserved names list
if bool(reserved_pattern.match(folder_name)):
if not auto_fix:
raise ValueError(f"tag: {folder_name} is reserved names in windows")
folder_name = "_" + folder_name
return folder_name
27 changes: 25 additions & 2 deletions swanlab/data/run/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
from datetime import datetime
from typing import Callable, Optional, Dict
from .operator import SwanLabRunOperator, RuntimeInfo
from ..formater import check_key_format
from ..formater import check_key_format, check_win_reserved_folder_name, check_unique_on_case_insensitive
from swanlab.env import get_mode, get_swanlog_dir
import random

from swankit.env import is_windows

class SwanLabRunState(Enum):
"""SwanLabRunState is an enumeration class that represents the state of the experiment.
Expand Down Expand Up @@ -259,6 +259,11 @@ def log(self, data: dict, step: int = None):
step : int, optional
The step number of the current data, if not provided, it will be automatically incremented.
If step is duplicated, the data will be ignored.

Raise
----------
ValueError:
Unsupported key names.
"""
if self.__state != SwanLabRunState.RUNNING:
raise RuntimeError("After experiment finished, you can no longer log data to the current experiment")
Expand All @@ -278,13 +283,31 @@ def log(self, data: dict, step: int = None):
step = None

log_return = {}
if is_windows():
# 在windows不区分大小写的情况下需要检查key是否大小写不敏感独一
check_unique_on_case_insensitive(data.keys())
# 遍历data,记录data
for k, v in data.items():
_k = k
k = check_key_format(k, auto_cut=True)
if k != _k:
# 超过255字符,截断
swanlog.warning(f"Key {_k} is too long, cut to 255 characters.")
if k in data.keys():
raise ValueError(f'tag: Unsupport too long Key "{_k}" and auto cut failed')
if is_windows():
# windows 中要增加保留文件夹名的判断
_k = k
k = check_win_reserved_folder_name(k)
if k != _k:
# key名为windows保留文件名
lower_key_name = [k.lower() in data.keys()]
if k.lower() in lower_key_name:
# 修复后又和原先key重名(大小写不区分情况下)
raise ValueError(
f"Key {_k} unsupport on windows and auto fix it failed. Please change a key name instead"
)
# swanlog.warning(f"Key {_k} unsupport on windows, auto used {k} instead") # todo: 每次都打很烦,暂时先注释掉,回头最好弄成只打一次warning
# ---------------------------------- 包装数据 ----------------------------------
# 输入为可转换为float的数据类型
if isinstance(v, (int, float, FloatConvertible)):
Expand Down