Tests

Adding PerTok to README.md #1

	# PyTest workflow

	name: Tests

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]

	jobs:
	test:
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	python-version: ["3.8", "3.9", "3.x"]
	os: [ ubuntu-latest, macos-latest, windows-latest ]

	steps:
	- uses: actions/checkout@v4

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: pip
	cache-dependency-path: pyproject.toml

	- name: Install dependencies
	run: \|
	# Install local package with tests dependencies extras
	python -m pip install --upgrade pip
	pip install -e ".[tests]"

	# Tokenizer training tests are significantly slower than others.
	# So that xdist don't assign chunks of training tests to the same worker, we use
	# the `--dist worksteal` distribution mode to dynamically reassign queued tests to
	# free workers.
	- name: Test with pytest
	run: pytest --cov=./ --cov-report=xml -n logical --dist worksteal --durations=0 -v tests
	env:
	HF_TOKEN_HUB_TESTS: ${{ secrets.HF_TOKEN_HUB_TESTS }}

	- name: Codecov
	uses: codecov/codecov-action@v4.6.0

	build:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.x'
	- name: Install dependencies
	run: pip install build
	- name: Build package
	run: python -m build

Provide feedback