mirror of
https://git.FreeBSD.org/ports.git
synced 2024-12-29 05:38:00 +00:00
textproc/py-tiktoken: Add py-tiktoken 0.4.0
tiktoken is a fast BPE tokeniser for use with OpenAI's models.
This commit is contained in:
Po-Chuan Hsieh
parent
d733ef5984
commit
6c34be9ab1
@ -1593,6 +1593,7 @@
|
||||
SUBDIR += py-texttable
|
||||
SUBDIR += py-textual
|
||||
SUBDIR += py-three-merge
|
||||
SUBDIR += py-tiktoken
|
||||
SUBDIR += py-tinycss
|
||||
SUBDIR += py-tinycss2
|
||||
SUBDIR += py-toml
|
||||
|
||||
35
textproc/py-tiktoken/Makefile
Normal file
35
textproc/py-tiktoken/Makefile
Normal file
@ -0,0 +1,35 @@
|
||||
PORTNAME= tiktoken
|
||||
PORTVERSION= 0.4.0
|
||||
CATEGORIES= textproc python
|
||||
MASTER_SITES= PYPI
|
||||
PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX}
|
||||
DISTFILES= ${DISTNAME}${EXTRACT_SUFX}
|
||||
|
||||
MAINTAINER= sunpoet@FreeBSD.org
|
||||
COMMENT= Fast BPE tokeniser for use with OpenAI's models
|
||||
WWW= https://github.com/openai/tiktoken
|
||||
|
||||
LICENSE= MIT
|
||||
LICENSE_FILE= ${WRKSRC}/LICENSE
|
||||
|
||||
BUILD_DEPENDS= ${PYTHON_PKGNAMEPREFIX}setuptools>=62.4:devel/py-setuptools@${PY_FLAVOR} \
|
||||
${PYTHON_PKGNAMEPREFIX}setuptools-rust>=1.5.2:devel/py-setuptools-rust@${PY_FLAVOR} \
|
||||
${PYTHON_PKGNAMEPREFIX}wheel>=0:devel/py-wheel@${PY_FLAVOR}
|
||||
RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}regex>=2022.1.18:textproc/py-regex@${PY_FLAVOR} \
|
||||
${PYTHON_PKGNAMEPREFIX}requests>=2.26.0:www/py-requests@${PY_FLAVOR}
|
||||
|
||||
USES= cargo python
|
||||
USE_PYTHON= autoplist concurrent pep517
|
||||
|
||||
CARGO_BUILD= no
|
||||
CARGO_INSTALL= no
|
||||
|
||||
OPTIONS_DEFINE= BLOBFILE
|
||||
BLOBFILE_DESC= Blob file support
|
||||
|
||||
BLOBFILE_RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}blobfile>=2:net/py-blobfile@${PY_FLAVOR}
|
||||
|
||||
post-install:
|
||||
${FIND} ${STAGEDIR}${PYTHON_SITELIBDIR} -name '*.so' -exec ${STRIP_CMD} {} +
|
||||
|
||||
.include <bsd.port.mk>
|
||||
43
textproc/py-tiktoken/Makefile.crates
Normal file
43
textproc/py-tiktoken/Makefile.crates
Normal file
@ -0,0 +1,43 @@
|
||||
CARGO_CRATES= aho-corasick-1.0.4 \
|
||||
autocfg-1.1.0 \
|
||||
bit-set-0.5.3 \
|
||||
bit-vec-0.6.3 \
|
||||
bitflags-1.3.2 \
|
||||
bstr-1.6.0 \
|
||||
cfg-if-1.0.0 \
|
||||
fancy-regex-0.10.0 \
|
||||
indoc-1.0.9 \
|
||||
libc-0.2.147 \
|
||||
lock_api-0.4.10 \
|
||||
memchr-2.5.0 \
|
||||
memoffset-0.6.5 \
|
||||
once_cell-1.18.0 \
|
||||
parking_lot-0.12.1 \
|
||||
parking_lot_core-0.9.8 \
|
||||
proc-macro2-1.0.66 \
|
||||
pyo3-0.17.3 \
|
||||
pyo3-build-config-0.17.3 \
|
||||
pyo3-ffi-0.17.3 \
|
||||
pyo3-macros-0.17.3 \
|
||||
pyo3-macros-backend-0.17.3 \
|
||||
quote-1.0.33 \
|
||||
redox_syscall-0.3.5 \
|
||||
regex-1.9.3 \
|
||||
regex-automata-0.3.6 \
|
||||
regex-syntax-0.7.4 \
|
||||
rustc-hash-1.1.0 \
|
||||
scopeguard-1.2.0 \
|
||||
serde-1.0.183 \
|
||||
smallvec-1.11.0 \
|
||||
syn-1.0.109 \
|
||||
target-lexicon-0.12.11 \
|
||||
unicode-ident-1.0.11 \
|
||||
unindent-0.1.11 \
|
||||
windows-targets-0.48.5 \
|
||||
windows_aarch64_gnullvm-0.48.5 \
|
||||
windows_aarch64_msvc-0.48.5 \
|
||||
windows_i686_gnu-0.48.5 \
|
||||
windows_i686_msvc-0.48.5 \
|
||||
windows_x86_64_gnu-0.48.5 \
|
||||
windows_x86_64_gnullvm-0.48.5 \
|
||||
windows_x86_64_msvc-0.48.5
|
||||
89
textproc/py-tiktoken/distinfo
Normal file
89
textproc/py-tiktoken/distinfo
Normal file
@ -0,0 +1,89 @@
|
||||
TIMESTAMP = 1692211619
|
||||
SHA256 (tiktoken-0.4.0.tar.gz) = 59b20a819969735b48161ced9b92f05dc4519c17be4015cfb73b65270a243620
|
||||
SIZE (tiktoken-0.4.0.tar.gz) = 25504
|
||||
SHA256 (rust/crates/aho-corasick-1.0.4.crate) = 6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a
|
||||
SIZE (rust/crates/aho-corasick-1.0.4.crate) = 171175
|
||||
SHA256 (rust/crates/autocfg-1.1.0.crate) = d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa
|
||||
SIZE (rust/crates/autocfg-1.1.0.crate) = 13272
|
||||
SHA256 (rust/crates/bit-set-0.5.3.crate) = 0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1
|
||||
SIZE (rust/crates/bit-set-0.5.3.crate) = 14470
|
||||
SHA256 (rust/crates/bit-vec-0.6.3.crate) = 349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb
|
||||
SIZE (rust/crates/bit-vec-0.6.3.crate) = 19927
|
||||
SHA256 (rust/crates/bitflags-1.3.2.crate) = bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a
|
||||
SIZE (rust/crates/bitflags-1.3.2.crate) = 23021
|
||||
SHA256 (rust/crates/bstr-1.6.0.crate) = 6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05
|
||||
SIZE (rust/crates/bstr-1.6.0.crate) = 385877
|
||||
SHA256 (rust/crates/cfg-if-1.0.0.crate) = baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd
|
||||
SIZE (rust/crates/cfg-if-1.0.0.crate) = 7934
|
||||
SHA256 (rust/crates/fancy-regex-0.10.0.crate) = 0678ab2d46fa5195aaf59ad034c083d351377d4af57f3e073c074d0da3e3c766
|
||||
SIZE (rust/crates/fancy-regex-0.10.0.crate) = 79951
|
||||
SHA256 (rust/crates/indoc-1.0.9.crate) = bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306
|
||||
SIZE (rust/crates/indoc-1.0.9.crate) = 13475
|
||||
SHA256 (rust/crates/libc-0.2.147.crate) = b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3
|
||||
SIZE (rust/crates/libc-0.2.147.crate) = 686772
|
||||
SHA256 (rust/crates/lock_api-0.4.10.crate) = c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16
|
||||
SIZE (rust/crates/lock_api-0.4.10.crate) = 26713
|
||||
SHA256 (rust/crates/memchr-2.5.0.crate) = 2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d
|
||||
SIZE (rust/crates/memchr-2.5.0.crate) = 65812
|
||||
SHA256 (rust/crates/memoffset-0.6.5.crate) = 5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce
|
||||
SIZE (rust/crates/memoffset-0.6.5.crate) = 7686
|
||||
SHA256 (rust/crates/once_cell-1.18.0.crate) = dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d
|
||||
SIZE (rust/crates/once_cell-1.18.0.crate) = 32969
|
||||
SHA256 (rust/crates/parking_lot-0.12.1.crate) = 3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f
|
||||
SIZE (rust/crates/parking_lot-0.12.1.crate) = 40967
|
||||
SHA256 (rust/crates/parking_lot_core-0.9.8.crate) = 93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447
|
||||
SIZE (rust/crates/parking_lot_core-0.9.8.crate) = 32383
|
||||
SHA256 (rust/crates/proc-macro2-1.0.66.crate) = 18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9
|
||||
SIZE (rust/crates/proc-macro2-1.0.66.crate) = 43575
|
||||
SHA256 (rust/crates/pyo3-0.17.3.crate) = 268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543
|
||||
SIZE (rust/crates/pyo3-0.17.3.crate) = 408939
|
||||
SHA256 (rust/crates/pyo3-build-config-0.17.3.crate) = 28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8
|
||||
SIZE (rust/crates/pyo3-build-config-0.17.3.crate) = 29230
|
||||
SHA256 (rust/crates/pyo3-ffi-0.17.3.crate) = 0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc
|
||||
SIZE (rust/crates/pyo3-ffi-0.17.3.crate) = 62854
|
||||
SHA256 (rust/crates/pyo3-macros-0.17.3.crate) = 94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28
|
||||
SIZE (rust/crates/pyo3-macros-0.17.3.crate) = 7257
|
||||
SHA256 (rust/crates/pyo3-macros-backend-0.17.3.crate) = c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f
|
||||
SIZE (rust/crates/pyo3-macros-backend-0.17.3.crate) = 49992
|
||||
SHA256 (rust/crates/quote-1.0.33.crate) = 5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae
|
||||
SIZE (rust/crates/quote-1.0.33.crate) = 28090
|
||||
SHA256 (rust/crates/redox_syscall-0.3.5.crate) = 567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29
|
||||
SIZE (rust/crates/redox_syscall-0.3.5.crate) = 23404
|
||||
SHA256 (rust/crates/regex-1.9.3.crate) = 81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a
|
||||
SIZE (rust/crates/regex-1.9.3.crate) = 252676
|
||||
SHA256 (rust/crates/regex-automata-0.3.6.crate) = fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69
|
||||
SIZE (rust/crates/regex-automata-0.3.6.crate) = 607174
|
||||
SHA256 (rust/crates/regex-syntax-0.7.4.crate) = e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2
|
||||
SIZE (rust/crates/regex-syntax-0.7.4.crate) = 343365
|
||||
SHA256 (rust/crates/rustc-hash-1.1.0.crate) = 08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2
|
||||
SIZE (rust/crates/rustc-hash-1.1.0.crate) = 9331
|
||||
SHA256 (rust/crates/scopeguard-1.2.0.crate) = 94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49
|
||||
SIZE (rust/crates/scopeguard-1.2.0.crate) = 11619
|
||||
SHA256 (rust/crates/serde-1.0.183.crate) = 32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c
|
||||
SIZE (rust/crates/serde-1.0.183.crate) = 77210
|
||||
SHA256 (rust/crates/smallvec-1.11.0.crate) = 62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9
|
||||
SIZE (rust/crates/smallvec-1.11.0.crate) = 34680
|
||||
SHA256 (rust/crates/syn-1.0.109.crate) = 72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237
|
||||
SIZE (rust/crates/syn-1.0.109.crate) = 237611
|
||||
SHA256 (rust/crates/target-lexicon-0.12.11.crate) = 9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a
|
||||
SIZE (rust/crates/target-lexicon-0.12.11.crate) = 24594
|
||||
SHA256 (rust/crates/unicode-ident-1.0.11.crate) = 301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c
|
||||
SIZE (rust/crates/unicode-ident-1.0.11.crate) = 42067
|
||||
SHA256 (rust/crates/unindent-0.1.11.crate) = e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c
|
||||
SIZE (rust/crates/unindent-0.1.11.crate) = 7700
|
||||
SHA256 (rust/crates/windows-targets-0.48.5.crate) = 9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c
|
||||
SIZE (rust/crates/windows-targets-0.48.5.crate) = 6904
|
||||
SHA256 (rust/crates/windows_aarch64_gnullvm-0.48.5.crate) = 2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8
|
||||
SIZE (rust/crates/windows_aarch64_gnullvm-0.48.5.crate) = 418492
|
||||
SHA256 (rust/crates/windows_aarch64_msvc-0.48.5.crate) = dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc
|
||||
SIZE (rust/crates/windows_aarch64_msvc-0.48.5.crate) = 798483
|
||||
SHA256 (rust/crates/windows_i686_gnu-0.48.5.crate) = a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e
|
||||
SIZE (rust/crates/windows_i686_gnu-0.48.5.crate) = 844891
|
||||
SHA256 (rust/crates/windows_i686_msvc-0.48.5.crate) = 8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406
|
||||
SIZE (rust/crates/windows_i686_msvc-0.48.5.crate) = 864300
|
||||
SHA256 (rust/crates/windows_x86_64_gnu-0.48.5.crate) = 53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e
|
||||
SIZE (rust/crates/windows_x86_64_gnu-0.48.5.crate) = 801619
|
||||
SHA256 (rust/crates/windows_x86_64_gnullvm-0.48.5.crate) = 0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc
|
||||
SIZE (rust/crates/windows_x86_64_gnullvm-0.48.5.crate) = 418486
|
||||
SHA256 (rust/crates/windows_x86_64_msvc-0.48.5.crate) = ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538
|
||||
SIZE (rust/crates/windows_x86_64_msvc-0.48.5.crate) = 798412
|
||||
1
textproc/py-tiktoken/pkg-descr
Normal file
1
textproc/py-tiktoken/pkg-descr
Normal file
@ -0,0 +1 @@
|
||||
tiktoken is a fast BPE tokeniser for use with OpenAI's models.
|
||||
Loading…
Reference in New Issue
Block a user