name: pyonmttok
description: Fast and customizable text tokenization library with BPE and SentencePiece support
license_spdx: MIT
version: 1.36.0
spec_version: 1.0
home_url: https://opennmt.net
news_url: https://libraries.io/pypi/pyonmttok/versions.atom
date: 2023-01-11
about_resource: NONE
libraries_io_url: https://libraries.io/pypi/pyonmttok?number=1.36.0
vcs_tool: git
vcs_repository: https://github.com/OpenNMT/Tokenizer
vcs_branch: master
