typst: add typst packages from typst universe
This commit is contained in:
Gongqi Huang
parent
8149e604ef
commit
d976d61d9e
226
maintainers/scripts/update-typst-packages.py
Executable file
226
maintainers/scripts/update-typst-packages.py
Executable file
@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3
|
||||
|
||||
# This file is formatted with `ruff format`.
|
||||
|
||||
import os
|
||||
import re
|
||||
import tomli
|
||||
import tomli_w
|
||||
import subprocess
|
||||
import concurrent.futures
|
||||
import argparse
|
||||
import tempfile
|
||||
import tarfile
|
||||
from string import punctuation
|
||||
from packaging.version import Version
|
||||
from urllib import request
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class TypstPackage:
|
||||
def __init__(self, **kwargs):
|
||||
self.pname = kwargs["pname"]
|
||||
self.version = kwargs["version"]
|
||||
self.meta = kwargs["meta"]
|
||||
self.path = kwargs["path"]
|
||||
self.repo = (
|
||||
None
|
||||
if "repository" not in self.meta["package"]
|
||||
else self.meta["package"]["repository"]
|
||||
)
|
||||
self.description = self.meta["package"]["description"].rstrip(punctuation)
|
||||
self.license = self.meta["package"]["license"]
|
||||
self.params = "" if "params" not in kwargs else kwargs["params"]
|
||||
self.deps = [] if "deps" not in kwargs else kwargs["deps"]
|
||||
|
||||
@classmethod
|
||||
def package_name_full(cls, package_name, version):
|
||||
version_number = map(lambda x: int(x), version.split("."))
|
||||
version_nix = "_".join(map(lambda x: str(x), version_number))
|
||||
return "_".join((package_name, version_nix))
|
||||
|
||||
def license_tokens(self):
|
||||
import license_expression as le
|
||||
|
||||
try:
|
||||
# FIXME: ad hoc conversion
|
||||
exception_list = [("EUPL-1.2+", "EUPL-1.2")]
|
||||
|
||||
def sanitize_license_string(license_string, lookups):
|
||||
if not lookups:
|
||||
return license_string
|
||||
return sanitize_license_string(
|
||||
license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:]
|
||||
)
|
||||
|
||||
sanitized = sanitize_license_string(self.license, exception_list)
|
||||
licensing = le.get_spdx_licensing()
|
||||
parsed = licensing.parse(sanitized, validate=True)
|
||||
return [s.key for s in licensing.license_symbols(parsed)]
|
||||
except le.ExpressionError as e:
|
||||
print(
|
||||
f'Failed to parse license string "{self.license}" because of {str(e)}'
|
||||
)
|
||||
exit(1)
|
||||
|
||||
def source(self):
|
||||
url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz"
|
||||
cmd = [
|
||||
"nix",
|
||||
"store",
|
||||
"prefetch-file",
|
||||
"--unpack",
|
||||
"--hash-type",
|
||||
"sha256",
|
||||
"--refresh",
|
||||
"--extra-experimental-features",
|
||||
"nix-command",
|
||||
]
|
||||
result = subprocess.run(cmd + [url], capture_output=True, text=True)
|
||||
hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0]
|
||||
return url, hash
|
||||
|
||||
def to_name_full(self):
|
||||
return self.package_name_full(self.pname, self.version)
|
||||
|
||||
def to_attrs(self):
|
||||
deps = set()
|
||||
excludes = list(map(
|
||||
lambda e: os.path.join(self.path, e),
|
||||
self.meta["package"]["exclude"] if "exclude" in self.meta["package"] else [],
|
||||
))
|
||||
for root, _, files in os.walk(self.path):
|
||||
for file in filter(lambda f: f.split(".")[-1] == "typ", files):
|
||||
file_path = os.path.join(root, file)
|
||||
if file_path in excludes:
|
||||
continue
|
||||
with open(file_path, "r") as f:
|
||||
deps.update(
|
||||
set(
|
||||
re.findall(
|
||||
r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"",
|
||||
f.read(),
|
||||
re.MULTILINE,
|
||||
)
|
||||
)
|
||||
)
|
||||
self.deps = list(
|
||||
filter(lambda p: p[0] != self.pname or p[1] != self.version, deps)
|
||||
)
|
||||
source_url, source_hash = self.source()
|
||||
|
||||
return dict(
|
||||
url=source_url,
|
||||
hash=source_hash,
|
||||
typstDeps=[
|
||||
self.package_name_full(p, v)
|
||||
for p, v in sorted(self.deps, key=lambda x: x[0])
|
||||
],
|
||||
description=self.description,
|
||||
license=self.license_tokens(),
|
||||
) | (dict(homepage=self.repo) if self.repo else dict())
|
||||
|
||||
|
||||
def generate_typst_packages(preview_dir, output_file):
|
||||
package_tree = dict()
|
||||
|
||||
print("Parsing metadata... from", preview_dir)
|
||||
for p in os.listdir(preview_dir):
|
||||
package_dir = os.path.join(preview_dir, p)
|
||||
for v in os.listdir(package_dir):
|
||||
package_version_dir = os.path.join(package_dir, v)
|
||||
with open(
|
||||
os.path.join(package_version_dir, "typst.toml"), "rb"
|
||||
) as meta_file:
|
||||
try:
|
||||
package = TypstPackage(
|
||||
pname=p,
|
||||
version=v,
|
||||
meta=tomli.load(meta_file),
|
||||
path=package_version_dir,
|
||||
)
|
||||
if package.pname in package_tree:
|
||||
package_tree[package.pname][v] = package
|
||||
else:
|
||||
package_tree[package.pname] = dict({v: package})
|
||||
except tomli.TOMLDecodeError:
|
||||
print("Invalid typst.toml:", package_version_dir)
|
||||
|
||||
with open(output_file, "wb") as typst_packages:
|
||||
|
||||
def generate_package(pname, package_subtree):
|
||||
sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True)
|
||||
print(f"Generating metadata for {pname}")
|
||||
return {
|
||||
pname: OrderedDict(
|
||||
(k, package_subtree[k].to_attrs()) for k in sorted_keys
|
||||
)
|
||||
}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
|
||||
sorted_packages = sorted(package_tree.items(), key=lambda x: x[0])
|
||||
futures = list()
|
||||
for pname, psubtree in sorted_packages:
|
||||
futures.append(executor.submit(generate_package, pname, psubtree))
|
||||
packages = OrderedDict(
|
||||
(package, subtree)
|
||||
for future in futures
|
||||
for package, subtree in future.result().items()
|
||||
)
|
||||
print(f"Writing metadata... to {output_file}")
|
||||
tomli_w.dump(packages, typst_packages)
|
||||
|
||||
|
||||
def main(args):
|
||||
PREVIEW_DIR = "packages/preview"
|
||||
TYPST_PACKAGE_TARBALL_URL = (
|
||||
"https://github.com/typst/packages/archive/refs/heads/main.tar.gz"
|
||||
)
|
||||
|
||||
directory = args.directory
|
||||
if not directory:
|
||||
tempdir = tempfile.mkdtemp()
|
||||
print(tempdir)
|
||||
typst_tarball = os.path.join(tempdir, "main.tar.gz")
|
||||
|
||||
print(
|
||||
"Downloading Typst packages source from {} to {}".format(
|
||||
TYPST_PACKAGE_TARBALL_URL, typst_tarball
|
||||
)
|
||||
)
|
||||
with request.urlopen(
|
||||
request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0
|
||||
) as response:
|
||||
if response.status == 200:
|
||||
with open(typst_tarball, "wb+") as f:
|
||||
f.write(response.read())
|
||||
else:
|
||||
print("Download failed")
|
||||
exit(1)
|
||||
with tarfile.open(typst_tarball) as tar:
|
||||
tar.extractall(path=tempdir, filter="data")
|
||||
directory = os.path.join(tempdir, "packages-main")
|
||||
directory = os.path.abspath(directory)
|
||||
|
||||
generate_typst_packages(
|
||||
os.path.join(directory, PREVIEW_DIR),
|
||||
args.output,
|
||||
)
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-d", "--directory", help="Local Typst Universe repository", default=None
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="Output file",
|
||||
default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
19718
pkgs/by-name/ty/typst/typst-packages-from-universe.toml
Normal file
19718
pkgs/by-name/ty/typst/typst-packages-from-universe.toml
Normal file
File diff suppressed because it is too large
Load Diff
52
pkgs/by-name/ty/typst/typst-packages.nix
Normal file
52
pkgs/by-name/ty/typst/typst-packages.nix
Normal file
@ -0,0 +1,52 @@
|
||||
{
|
||||
lib,
|
||||
callPackage,
|
||||
}:
|
||||
|
||||
let
|
||||
toPackageName = name: version: "${name}_${lib.replaceStrings [ "." ] [ "_" ] version}";
|
||||
in
|
||||
lib.makeExtensible (
|
||||
final:
|
||||
lib.recurseIntoAttrs (
|
||||
lib.foldlAttrs (
|
||||
packageSet: pname: versionSet:
|
||||
packageSet
|
||||
// (lib.foldlAttrs (
|
||||
subPackageSet: version: packageSpec:
|
||||
subPackageSet
|
||||
// {
|
||||
${toPackageName pname version} = callPackage (
|
||||
{
|
||||
lib,
|
||||
buildTypstPackage,
|
||||
fetchzip,
|
||||
}:
|
||||
buildTypstPackage (finalAttrs: {
|
||||
inherit pname version;
|
||||
|
||||
src = fetchzip {
|
||||
inherit (packageSpec) hash;
|
||||
url = "https://packages.typst.org/preview/${finalAttrs.pname}-${finalAttrs.version}.tar.gz";
|
||||
stripRoot = false;
|
||||
};
|
||||
|
||||
typstDeps = builtins.filter (x: x != null) (
|
||||
lib.map (d: (lib.attrsets.attrByPath [ d ] null final)) packageSpec.typstDeps
|
||||
);
|
||||
|
||||
meta = {
|
||||
inherit (packageSpec) description;
|
||||
maintainers = with lib.maintainers; [ cherrypiejam ];
|
||||
license = lib.map (lib.flip lib.getAttr lib.licensesSpdx) packageSpec.license;
|
||||
} // (if packageSpec ? "homepage" then { inherit (packageSpec) homepage; } else { });
|
||||
})
|
||||
) { };
|
||||
}
|
||||
) { } versionSet)
|
||||
// {
|
||||
${pname} = final.${toPackageName pname (lib.last (lib.attrNames versionSet))};
|
||||
}
|
||||
) { } (lib.importTOML ./typst-packages-from-universe.toml)
|
||||
)
|
||||
)
|
||||
Loading…
x
Reference in New Issue
Block a user