incus: build and expose documentation (#395767)

nixos/modules/virtualisation/incus.nix

@@ -129,6 +129,7 @@ let
 
   environment = lib.mkMerge [
     {
+      INCUS_DOCUMENTATION = "${cfg.package.doc}/html";
       INCUS_EDK2_PATH = ovmf;
       INCUS_LXC_HOOK = "${cfg.lxcPackage}/share/lxc/hooks";
       INCUS_LXC_TEMPLATE_CONFIG = "${pkgs.lxcfs}/share/lxc/config";

nixos/tests/incus/ui.nix

@@ -66,6 +66,7 @@ import ../make-test-python.nix (
 
     testScript = ''
       machine.wait_for_unit("incus.service")
+      machine.wait_for_unit("incus-preseed.service")
 
       # Check that the INCUS_UI environment variable is populated in the systemd unit
       machine.succeed("systemctl cat incus.service | grep 'INCUS_UI'")
@@ -73,6 +74,9 @@ import ../make-test-python.nix (
       # Ensure the endpoint returns an HTML page with 'Incus UI' in the title
       machine.succeed("curl -kLs https://localhost:8443/ui | grep '<title>Incus UI</title>'")
 
+      # Ensure the documentation is rendering correctly
+      machine.succeed("curl -kLs https://localhost:8443/documentation/ | grep '<title>Incus documentation</title>'")
+
       # Ensure the application is actually rendered by the Javascript
       machine.succeed("PYTHONUNBUFFERED=1 selenium-script")
     '';

pkgs/by-name/in/incus/docs.patch

@@ -0,0 +1,26 @@
+diff --git i/doc/conf.py w/doc/conf.py
+index 8d042818b..b4f0572bd 100644
+--- i/doc/conf.py
++++ w/doc/conf.py
+@@ -8,10 +8,6 @@ import yaml
+ from git import Repo
+ import filecmp
+ 
+-# Download and link swagger-ui files
+-if not os.path.isdir('.sphinx/deps/swagger-ui'):
+-    Repo.clone_from('https://github.com/swagger-api/swagger-ui', '.sphinx/deps/swagger-ui', depth=1)
+-
+ os.makedirs('.sphinx/_static/swagger-ui/', exist_ok=True)
+ 
+ if not os.path.islink('.sphinx/_static/swagger-ui/swagger-ui-bundle.js'):
+@@ -151,10 +147,6 @@ if os.path.exists("./related_topics.yaml"):
+     with open("./related_topics.yaml", "r") as fd:
+         myst_substitutions.update(yaml.safe_load(fd.read()))
+ 
+-intersphinx_mapping = {
+-    'cloud-init': ('https://cloudinit.readthedocs.io/en/latest/', None)
+-}
+-
+ if ("LOCAL_SPHINX_BUILD" in os.environ) and (os.environ["LOCAL_SPHINX_BUILD"] == "True"):
+     swagger_url_scheme = "/api/#{{path}}"
+ else:

pkgs/by-name/in/incus/generic.nix

@@ -13,6 +13,7 @@
   buildGoModule,
   fetchFromGitHub,
   acl,
+  buildPackages,
   cowsql,
   incus-ui-canonical,
   libcap,
@@ -27,11 +28,30 @@
 
 let
   pname = "incus${lib.optionalString lts "-lts"}";
+  docsPython = buildPackages.python3.withPackages (
+    py: with py; [
+      furo
+      gitpython
+      linkify-it-py
+      canonical-sphinx-extensions
+      myst-parser
+      pyspelling
+      sphinx
+      sphinx-autobuild
+      sphinx-copybutton
+      sphinx-design
+      sphinx-notfound-page
+      sphinx-remove-toctrees
+      sphinx-reredirects
+      sphinx-tabs
+      sphinxcontrib-jquery
+      sphinxext-opengraph
+    ]
+  );
 in
 
-buildGoModule rec {
+buildGoModule (finalAttrs: {
   inherit
-    patches
     pname
     vendorHash
     version
@@ -40,15 +60,18 @@ buildGoModule rec {
   outputs = [
     "out"
     "agent_loader"
+    "doc"
   ];
 
   src = fetchFromGitHub {
     owner = "lxc";
     repo = "incus";
-    rev = "refs/tags/v${version}";
+    tag = "v${version}";
     inherit hash;
   };
 
+  patches = [ ./docs.patch ] ++ patches;
+
   excludedPackages = [
     # statically compile these
     "cmd/incus-agent"
@@ -61,6 +84,7 @@ buildGoModule rec {
   nativeBuildInputs = [
     installShellFiles
     pkg-config
+    docsPython
   ];
 
   buildInputs = [
@@ -82,6 +106,13 @@ buildGoModule rec {
   CGO_LDFLAGS_ALLOW = "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)";
 
   postBuild = ''
+    # build docs
+    mkdir -p .sphinx/deps
+    ln -s ${buildPackages.python3.pkgs.swagger-ui-bundle.src} .sphinx/deps/swagger-ui
+    substituteInPlace Makefile --replace-fail '. $(SPHINXENV) ; ' ""
+    make doc-incremental
+
+    # build some static executables
     make incus-agent incus-migrate
   '';
 
@@ -111,18 +142,20 @@ buildGoModule rec {
     cp internal/server/instance/drivers/agent-loader/systemd/incus-agent.service $agent_loader/etc/systemd/system/
     cp internal/server/instance/drivers/agent-loader/systemd/incus-agent.rules $agent_loader/lib/udev/rules.d/99-incus-agent.rules
     substituteInPlace $agent_loader/etc/systemd/system/incus-agent.service --replace-fail 'TARGET/systemd' "$agent_loader/bin"
+
+    mkdir $doc
+    cp -R doc/html $doc/
   '';
 
   passthru = {
     client = callPackage ./client.nix {
       inherit
         lts
-        meta
         patches
-        src
         vendorHash
         version
         ;
+      inherit (finalAttrs) meta src;
     };
 
     tests = if lts then nixosTests.incus-lts.all else nixosTests.incus.all;
@@ -143,4 +176,4 @@ buildGoModule rec {
     platforms = lib.platforms.linux;
     mainProgram = "incus";
   };
-}
+})

pkgs/development/python-modules/canonical-sphinx-extensions/default.nix

@@ -0,0 +1,44 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchPypi,
+  setuptools,
+  beautifulsoup4,
+  docutils,
+  gitpython,
+  requests,
+  sphinx,
+}:
+
+buildPythonPackage rec {
+  pname = "canonical-sphinx-extensions";
+  version = "0.0.27";
+  pyproject = true;
+
+  src = fetchPypi {
+    pname = "canonical_sphinx_extensions";
+    inherit version;
+    hash = "sha256-ZorSmn+PAVS8xO7X3zk6u3W7pn3JB9w0PhFAXzv6l78=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    beautifulsoup4
+    docutils
+    gitpython
+    requests
+    sphinx
+  ];
+
+  doCheck = false;
+
+  meta = {
+    description = "A collection of Sphinx extensions used by Canonical documentation";
+    homepage = "https://pypi.org/project/canonical-sphinx-extensions";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ ];
+  };
+}

pkgs/development/python-modules/pyspelling/default.nix

@@ -0,0 +1,49 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchPypi,
+  hatchling,
+  beautifulsoup4,
+  html5lib,
+  lxml,
+  markdown,
+  pyyaml,
+  soupsieve,
+  wcmatch,
+}:
+
+buildPythonPackage rec {
+  pname = "pyspelling";
+  version = "2.10";
+  pyproject = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-rNZxM8G3zs1BDj1EieYfLksfC2rPGubEjCQPuyFynDc=";
+  };
+
+  build-system = [
+    hatchling
+  ];
+
+  dependencies = [
+    beautifulsoup4
+    html5lib
+    lxml
+    markdown
+    pyyaml
+    soupsieve
+    wcmatch
+  ];
+
+  pythonImportsCheck = [
+    "pyspelling"
+  ];
+
+  meta = {
+    description = "Spell checker";
+    homepage = "https://pypi.org/project/pyspelling";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ ];
+  };
+}

pkgs/development/python-modules/sphinx-remove-toctrees/default.nix

@@ -0,0 +1,60 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchPypi,
+  hatchling,
+  sphinx,
+  pre-commit,
+  ipython,
+  myst-parser,
+  sphinx-book-theme,
+  pytest,
+}:
+
+buildPythonPackage rec {
+  pname = "sphinx-remove-toctrees";
+  version = "1.0.0.post1";
+  pyproject = true;
+
+  src = fetchPypi {
+    pname = "sphinx_remove_toctrees";
+    inherit version;
+    hash = "sha256-SAjR7fFRwG7/bSw5Iux+vJ/Tqhdi3hsuFnSjf1rJzi0=";
+  };
+
+  build-system = [
+    hatchling
+  ];
+
+  dependencies = [
+    sphinx
+  ];
+
+  optional-dependencies = {
+    code_style = [
+      pre-commit
+    ];
+    docs = [
+      ipython
+      myst-parser
+      sphinx-book-theme
+    ];
+    tests = [
+      ipython
+      myst-parser
+      pytest
+      sphinx-book-theme
+    ];
+  };
+
+  pythonImportsCheck = [
+    "sphinx_remove_toctrees"
+  ];
+
+  meta = {
+    description = "Reduce your documentation build size by selectively removing toctrees from pages";
+    homepage = "https://pypi.org/project/sphinx-remove-toctrees/";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ ];
+  };
+}

pkgs/development/python-modules/sphinx-reredirects/default.nix

@@ -0,0 +1,41 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchPypi,
+  setuptools,
+  sphinx,
+}:
+
+buildPythonPackage rec {
+  pname = "sphinx-reredirects";
+  version = "0.1.6";
+  pyproject = true;
+
+  src = fetchPypi {
+    pname = "sphinx_reredirects";
+    inherit version;
+    hash = "sha256-xJHLpUX2e+lpdQhyeBjYYmYmNmJFrmRFb+KfN+m76mQ=";
+  };
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    sphinx
+  ];
+
+  pythonImportsCheck = [
+    "sphinx_reredirects"
+  ];
+
+  meta = {
+    description = "Handles redirects for moved pages in Sphinx documentation projects";
+    homepage = "https://pypi.org/project/sphinx-reredirects";
+    license = with lib.licenses; [
+      bsd3
+      mit
+    ];
+    maintainers = with lib.maintainers; [ ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -2192,6 +2192,10 @@ self: super: with self; {
 
   canmatrix = callPackage ../development/python-modules/canmatrix { };
 
+  canonical-sphinx-extensions =
+    callPackage ../development/python-modules/canonical-sphinx-extensions
+      { };
+
   canonicaljson = callPackage ../development/python-modules/canonicaljson { };
 
   canopen = callPackage ../development/python-modules/canopen { };
@@ -13320,6 +13324,8 @@ self: super: with self; {
 
   pyspellchecker = callPackage ../development/python-modules/pyspellchecker { };
 
+  pyspelling = callPackage ../development/python-modules/pyspelling { };
+
   pyspf = callPackage ../development/python-modules/pyspf { };
 
   pyspice = callPackage ../development/python-modules/pyspice { };
@@ -16165,6 +16171,10 @@ self: super: with self; {
 
   sphinx-pytest = callPackage ../development/python-modules/sphinx-pytest { };
 
+  sphinx-reredirects = callPackage ../development/python-modules/sphinx-reredirects { };
+
+  sphinx-remove-toctrees = callPackage ../development/python-modules/sphinx-remove-toctrees { };
+
   sphinx-rtd-dark-mode = callPackage ../development/python-modules/sphinx-rtd-dark-mode { };
 
   sphinx-rtd-theme = callPackage ../development/python-modules/sphinx-rtd-theme { };