65b802a6fd
It's disabled on MacOS by default: https://pytorch.org/docs/stable/distributed.html#torch.distributed.is_available When distributed module build is enabled, a compilation error happens on fresh clang. Upstream repo (which is a submodule in pytorch) is archived so it's impossible to contribute the fix there. Instead, an inline patch is included that is enough to fix the error. Signed-off-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com>
79 lines
1.5 KiB
Nix
79 lines
1.5 KiB
Nix
{
|
|
lib,
|
|
buildPythonPackage,
|
|
fetchPypi,
|
|
|
|
# build-system
|
|
setuptools,
|
|
|
|
# dependencies
|
|
bitarray,
|
|
datasets,
|
|
flask,
|
|
python-dotenv,
|
|
ninja,
|
|
scipy,
|
|
tqdm,
|
|
transformers,
|
|
ujson,
|
|
gitpython,
|
|
torch,
|
|
faiss,
|
|
}:
|
|
|
|
buildPythonPackage rec {
|
|
pname = "colbert-ai";
|
|
version = "0.2.21";
|
|
pyproject = true;
|
|
|
|
src = fetchPypi {
|
|
inherit version;
|
|
pname = "colbert_ai";
|
|
hash = "sha256-qNb9tOInLysI7Tf45QlgchYNhBXR5AWFdRiYt35iW6s=";
|
|
};
|
|
|
|
# ImportError: cannot import name 'AdamW' from 'transformers'
|
|
# https://github.com/stanford-futuredata/ColBERT/pull/390
|
|
postPatch = ''
|
|
substituteInPlace colbert/training/training.py \
|
|
--replace-fail \
|
|
"from transformers import AdamW, get_linear_schedule_with_warmup" \
|
|
"from transformers import get_linear_schedule_with_warmup; from torch.optim import AdamW"
|
|
'';
|
|
|
|
pythonRemoveDeps = [ "git-python" ];
|
|
|
|
build-system = [
|
|
setuptools
|
|
];
|
|
|
|
dependencies = [
|
|
bitarray
|
|
datasets
|
|
faiss
|
|
flask
|
|
gitpython
|
|
python-dotenv
|
|
ninja
|
|
scipy
|
|
torch
|
|
tqdm
|
|
transformers
|
|
ujson
|
|
];
|
|
|
|
pythonImportsCheck = [ "colbert" ];
|
|
|
|
# There is no tests
|
|
doCheck = false;
|
|
|
|
meta = {
|
|
description = "Fast and accurate retrieval model, enabling scalable BERT-based search over large text collections in tens of milliseconds";
|
|
homepage = "https://github.com/stanford-futuredata/ColBERT";
|
|
license = lib.licenses.mit;
|
|
maintainers = with lib.maintainers; [
|
|
bachp
|
|
];
|
|
};
|
|
}
|