levanter/pyproject.toml at main · autoGLM/levanter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "levanter"
version = "1.1"
authors = [
  { name="David Hall", email="dlwh@cs.stanford.edu" },
  {name="Ivan Zhou", email="ivanz@stanford.edu"}
]
description = "Scalable Training for Foundation Models with Named Tensors and JAX"
readme = "README.md"
requires-python = ">=3.10"
classifiers = [
    "Programming Language :: Python :: 3",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: POSIX :: Linux",
    "Operating System :: MacOS :: MacOS X",
    "Development Status :: 4 - Beta",
    "Intended Audience :: Science/Research",
]
dependencies = [
    # we require that you install jax yourself, since the extras vary by system.
    # jax = {version = ">=0.4.10,<0.5.0"}
#    "haliax>=1.3,<2.0",
#     Haliax changes in step with levanter, so we'll just use the git version except for releases.
#    "haliax @ git+https://github.com/stanford-crfm/haliax.git@main",
    "haliax>=1.4.dev301",
    "equinox>=0.11.4",
    "jaxtyping>=0.2.20",
    "tokenizers>=0.15.2",
    "transformers>=4.41.2",
    "optax>=0.1.9",
    "wandb>=0.16.6,<0.18.0",
    # We don't actually directly depend on scipy, but recent JAX had an issue
    "scipy<=1.12.0",
    "draccus>=0.8.0",
    "pyarrow>=11.0.0",
    "zstandard>=0.20.0",
    "datasets~=2.18",
    "gcsfs>=2024.2,<2024.7",
    "braceexpand>=0.1.7",
    "jmp>=0.0.3",
    "fsspec[http]>=2024.2,<2024.7",
    "tensorstore>=0.1.62",
    "pytimeparse>=1.1.8",
    "humanfriendly==10.0",
    "safetensors[numpy]~=0.4.2",
    "matplotlib>=3.7.0",
    "tblib>=1.7.0,<4.0.0",
    "dataclasses-json~=0.6.4",
    "ray[default]~=2.10",
    "pydantic<3",  # temporary pin until Ray supports pydantic 2.0
    "rich~=13.0",
    "filelock~=3.13",
]

[tool.hatch.build]
include = ["config/*.yaml", "config/*/*.yaml", "*.py"]
dev-mode-dirs = [".", "src"]

[tool.hatch.build.sources]
"src/levanter" = "levanter"
"config" = "levanter/config"

[tool.hatch.metadata]
allow-direct-references = true

[tool.hatch.build.targets.wheel]
packages = ["levanter"]


[project.urls]
"Homepage" = "https://github.com/stanford-crfm/levanter"
"Bug Tracker" = "https://github.com/stanford-crfm/levanter/issues"


[tool.black]
line-length = 119
target-version = ["py310"]
preview = true

[tool.isort]
profile = "black"
multi_line_output = 3
lines_after_imports = 2
include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
ensure_newline_before_comments = true
line_length = 119
src_paths = ["src", "tests"]
known_haliax = ["haliax"]
sections=["FUTURE", "STDLIB", "THIRDPARTY", "HALIAX", "FIRSTPARTY", "LOCALFOLDER"]

[tool.mypy]
python_version = "3.10"
mypy_path = ["src"]

[tool.mypy-haliax.core]
ignore_missing_imports = true

[tool.pytest.ini_options]
pythonpath = [ "tests" ]
markers = [
    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
    "entry: marks tests as entry point tests (deselect with '-m \"not entry\"')",
    "ray: marks tests that require Ray (deselect with '-m \"not ray\"')",
]

[project.optional-dependencies]
test = [
    "pytest",
    "flake8",
    "pytest",
    "soundfile",
    "librosa",
    "pytest-forked"
]