doc/manpages: add the man pages to the docs

.gitignore

@@ -20,6 +20,7 @@ test/syscall/sysinfo/sysinfo
 
 # Sphinx
 doc/html/
+doc/reference/manpages/**/*.md
 doc/config_options.txt
 doc/config_options.yaml
 doc/.sphinx/deps/

Makefile

@@ -114,10 +114,11 @@ endif
 	swagger generate spec -o doc/rest-api.yaml -w ./lxd -m
 
 .PHONY: doc-setup
-doc-setup:
+doc-setup: client
 	@echo "Setting up documentation build environment"
 	python3 -m venv doc/.sphinx/venv
 	. $(SPHINXENV) ; pip install --upgrade -r doc/.sphinx/requirements.txt
+	find doc/reference/manpages/ -name "*.md" -type f -delete
 	rm -Rf doc/html
 
 .PHONY: generate-config
@@ -131,7 +132,7 @@ doc: doc-setup doc-incremental
 .PHONY: doc-incremental
 doc-incremental:
 	@echo "Build the documentation"
-	. $(SPHINXENV) ; sphinx-build -c doc/ -b dirhtml doc/ doc/html/ -w doc/.sphinx/warnings.txt
+	. $(SPHINXENV) ; LOCAL_SPHINX_BUILD=True sphinx-build -c doc/ -b dirhtml doc/ doc/html/ -w doc/.sphinx/warnings.txt
 
 .PHONY: doc-serve
 doc-serve:
@@ -143,7 +144,7 @@ doc-spellcheck: doc
 
 .PHONY: doc-linkcheck
 doc-linkcheck: doc-setup
-	. $(SPHINXENV) ; sphinx-build -c doc/ -b linkcheck doc/ doc/html/
+	. $(SPHINXENV) ; LOCAL_SPHINX_BUILD=True sphinx-build -c doc/ -b linkcheck doc/ doc/html/
 
 .PHONY: doc-lint
 doc-lint:

doc/.readthedocs.yaml

@@ -11,6 +11,9 @@ build:
   tools:
     golang: "1.18"
     python: "3.11"
+  jobs:
+    pre_build:
+      - go build -o lxc.bin ./lxc
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

doc/.sphinx/.markdownlint/doc-lint.sh

@@ -14,6 +14,8 @@ for fn in $(find doc/ -name '*.md'); do
     sed -E "s/(\(.+\)=)/\1\n/" $fn > .tmp/$fn;
 done
 
+rm -rf .tmp/doc/reference/manpages/
+
 mdl .tmp/doc -sdoc/.sphinx/.markdownlint/style.rb -udoc/.sphinx/.markdownlint/rules.rb --ignore-front-matter > .tmp/errors.txt || true
 
 ## Postprocessing

doc/.sphinx/.spellcheck.yaml

@@ -8,7 +8,7 @@ matrix:
     - doc/.sphinx/.wordlist.txt
     output: doc/.sphinx/.wordlist.dic
   sources:
-  - doc/html/**/*.html|!doc/html/config-options/index.html
+  - doc/html/**/*.html|!doc/html/config-options/index.html|!doc/html/reference/manpages/**/*.html
   pipeline:
   - pyspelling.filters.html:
       comments: false
@@ -24,3 +24,4 @@ matrix:
       - div.relatedlinks
       - span.guilabel
       - a.p-navigation__link
+      - div.visually-hidden

doc/api-extensions.md

@@ -667,7 +667,7 @@ Those keys control whether to put the LXD rules before or after any pre-existing
 This introduces a new `recursion=2` mode for `GET /1.0/containers` which allows for the retrieval of
 all container structs, including the state, snapshots and backup structs.
 
-This effectively allows for `lxc list` to get all it needs in one query.
+This effectively allows for [`lxc list`](lxc_list.md) to get all it needs in one query.
 
 ## `candid_authentication`
 
@@ -683,7 +683,7 @@ allows configuration of backup compression.
 
 This introduces the configuration keys `candid.domains` and `candid.expiry`. The
 former allows specifying allowed/valid Candid domains, the latter makes the
-macaroon's expiry configurable. The `lxc remote add` command now has a
+macaroon's expiry configurable. The [`lxc remote add`](lxc_remote_add.md) command now has a
 `--domain` flag which allows specifying a Candid domain.
 
 ## `nvidia_runtime_config`
@@ -801,7 +801,7 @@ parts have to be used.
 
 Snapshots which are then created will be given an expiry date based on the
 expression. This expiry date, defined by `expires_at`, can be manually edited
-using the API or `lxc config edit`. Snapshots with a valid expiry date will be
+using the API or [`lxc config edit`](lxc_config_edit.md). Snapshots with a valid expiry date will be
 removed when the task in run. Expiry can be disabled by setting `expires_at` to
 an empty string or `0001-01-01T00:00:00Z` (zero time). This is the default if
 `snapshots.expiry` is not set.
@@ -895,7 +895,7 @@ decide to trigger various actions.
 
 ## `lxc_features`
 
-This introduces the `lxc_features` section output from the `lxc info` command
+This introduces the `lxc_features` section output from the [`lxc info`](lxc_info.md) command
 via the `GET /1.0` route. It outputs the result of checks for key features being present in the
 underlying LXC library.
 
@@ -2248,7 +2248,7 @@ Introduce a new `security.csm` configuration key to control the use of
 be run in LXD VMs.
 
 ## `instances_rebuild`
-This extension adds the ability to rebuild an instance with the same origin image, alternate image or as empty. A new `POST /1.0/instances/<name>/rebuild?project=<project>` API endpoint has been added as well as a new CLI command `lxc rebuild`.
+This extension adds the ability to rebuild an instance with the same origin image, alternate image or as empty. A new `POST /1.0/instances/<name>/rebuild?project=<project>` API endpoint has been added as well as a new CLI command [`lxc rebuild`](lxc_rebuild.md).
 
 ## `numa_cpu_placement`
 This adds the possibility to place a set of CPUs in a desired set of NUMA nodes.
@@ -2260,12 +2260,12 @@ This adds the following new configuration key:
 ## `custom_volume_iso`
 This adds the possibility to import ISO images as custom storage volumes.
 
-This adds the `--type` flag to `lxc storage volume import`.
+This adds the `--type` flag to [`lxc storage volume import`](lxc_storage_volume_import.md).
 
 ## `network_allocations`
 This adds the possibility to list a LXD deployment's network allocations.
 
-Through the `lxc network list-allocations` command and the `--project <PROJECT> | --all-projects` flags,
+Through the [`lxc network list-allocations`](lxc_network_list-allocations.md) command and the `--project <PROJECT> | --all-projects` flags,
 you can list all the used IP addresses, hardware addresses (for instances), resource URIs and whether it uses NAT for
 each `instance`, `network`, `network forward` and `network load-balancer`.
 

doc/authentication.md

@@ -48,7 +48,7 @@ any backward compatibility to broken protocol or ciphers.
 (authentication-trusted-clients)=
 ### Trusted TLS clients
 
-You can obtain the list of TLS certificates trusted by a LXD server with `lxc config trust list`.
+You can obtain the list of TLS certificates trusted by a LXD server with [`lxc config trust list`](lxc_config_trust_list.md).
 
 Trusted clients can be added in either of the following ways:
 
@@ -58,7 +58,7 @@ Trusted clients can be added in either of the following ways:
 
 The workflow to authenticate with the server is similar to that of SSH, where an initial connection to an unknown server triggers a prompt:
 
-1. When the user adds a server with `lxc remote add`, the server is contacted over HTTPS, its certificate is downloaded and the fingerprint is shown to the user.
+1. When the user adds a server with [`lxc remote add`](lxc_remote_add.md), the server is contacted over HTTPS, its certificate is downloaded and the fingerprint is shown to the user.
 1. The user is asked to confirm that this is indeed the server's fingerprint, which they can manually check by connecting to the server or by asking someone with access to the server to run the info command and compare the fingerprints.
 1. The server attempts to authenticate the client:
 
@@ -67,20 +67,20 @@ The workflow to authenticate with the server is similar to that of SSH, where an
      If the provided token or trust password matches, the client certificate is added to the server's trust store and the connection is granted.
      Otherwise, the connection is rejected.
 
-To revoke trust to a client, remove its certificate from the server with `lxc config trust remove FINGERPRINT`.
+To revoke trust to a client, remove its certificate from the server with [`lxc config trust remove <fingerprint>`](lxc_config_trust_remove.md).
 
 It's possible to restrict a TLS client to one or multiple projects.
 In this case, the client will also be prevented from performing global configuration changes or altering the configuration (limits, restrictions) of the projects it's allowed access to.
 
-To restrict access, use `lxc config trust edit FINGERPRINT`.
+To restrict access, use [`lxc config trust edit <fingerprint>`](lxc_config_trust_edit.md).
 Set the `restricted` key to `true` and specify a list of projects to restrict the client to.
 If the list of projects is empty, the client will not be allowed access to any of them.
 
 (authentication-add-certs)=
 #### Adding trusted certificates to the server
 
 The preferred way to add trusted clients is to directly add their certificates to the trust store on the server.
-To do so, copy the client certificate to the server and register it using `lxc config trust add <file>`.
+To do so, copy the client certificate to the server and register it using [`lxc config trust add <file>`](lxc_config_trust_add.md).
 
 (authentication-trust-pw)=
 #### Adding client certificates using a trust password
@@ -95,7 +95,7 @@ This prevents brute-force attacks trying to guess the password.
 
 You can also add new clients by using tokens. This is a safer way than using the trust password, because tokens expire after a configurable time ({config:option}`server-core:core.remote_token_expiry`) or once they've been used.
 
-To use this method, generate a token for each client by calling `lxc config trust add`, which will prompt for the client name.
+To use this method, generate a token for each client by calling [`lxc config trust add`](lxc_config_trust_add.md), which will prompt for the client name.
 The clients can then add their certificates to the server's trust store by providing the generated token when prompted for the trust password.
 
 <!-- Include start NAT authentication -->
@@ -114,7 +114,7 @@ In this case, you must specify the external address manually.
 
 <!-- Include end NAT authentication -->
 
-Alternatively, the clients can provide the token directly when adding the remote: `lxc remote add <name> <token>`.
+Alternatively, the clients can provide the token directly when adding the remote: [`lxc remote add <name> <token>`](lxc_remote_add.md).
 
 ### Using a PKI system
 
@@ -151,7 +151,7 @@ Any user that authenticates through the configured OIDC Identity Provider gets f
 To configure LXD to use OIDC authentication, set the [`oidc.*`](server-options-oidc) server configuration options.
 Your OIDC provider must be configured to enable the [Device Authorization Grant](https://oauth.net/2/device-flow/) type.
 
-To add a remote pointing to a LXD server configured with OIDC authentication, run `lxc remote add <remote_name> <remote_address>`.
+To add a remote pointing to a LXD server configured with OIDC authentication, run [`lxc remote add <remote_name> <remote_address>`](lxc_remote_add.md).
 You are then prompted to authenticate through your web browser, where you must confirm the device code that LXD uses.
 The LXD client then retrieves and stores the access and refresh tokens and provides those to LXD for all interactions.
 
@@ -166,7 +166,7 @@ In this case, clients that try to authenticate with the server must get a Discha
 
 The authentication server certificate must be trusted by the LXD server.
 
-To add a remote pointing to a LXD server configured with Candid/Macaroon authentication, run `lxc remote add REMOTE ENDPOINT --auth-type=candid`.
+To add a remote pointing to a LXD server configured with Candid/Macaroon authentication, run [`lxc remote add REMOTE ENDPOINT --auth-type=candid`](lxc_remote_add.md).
 To verify the user, the client will prompt for the credentials required by the authentication server.
 If the authentication is successful, the client will connect to the LXD server and present the token received from the authentication server.
 The LXD server verifies the token, thus authenticating the request.

doc/cloud-init.md

@@ -70,7 +70,7 @@ To configure `cloud-init` for an instance, add the corresponding configuration o
 
 When configuring `cloud-init` directly for an instance, keep in mind that `cloud-init` runs only on the first start of the instance.
 That means that you must configure `cloud-init` before you start the instance.
-To do so, create the instance with `lxc init` instead of `lxc launch`, and then start it after completing the configuration.
+To do so, create the instance with `lxc init` instead of [`lxc launch`](lxc_launch.md), and then start it after completing the configuration.
 
 ### YAML format for `cloud-init` configuration
 

doc/conf.py

@@ -1,12 +1,13 @@
 import contextlib
 import datetime
 import os
+import stat
 import subprocess
-import sys
 import tempfile
 import yaml
 from git import Repo
 import wget
+import filecmp
 
 # Download and link swagger-ui files
 if not os.path.isdir('.sphinx/deps/swagger-ui'):
@@ -21,6 +22,89 @@
 if not os.path.islink('.sphinx/_static/swagger-ui/swagger-ui.css'):
     os.symlink('../../deps/swagger-ui/dist/swagger-ui.css', '.sphinx/_static/swagger-ui/swagger-ui.css')
 
+### MAN PAGES ###
+
+# Find path to lxc client (different for local builds and on RTD)
+
+if ("LOCAL_SPHINX_BUILD" in os.environ and
+    os.environ["LOCAL_SPHINX_BUILD"] == "True"):
+    path = str(subprocess.check_output(['go', 'env', 'GOPATH'], encoding="utf-8").strip())
+    lxc = os.path.join(path, 'bin', 'lxc')
+    if os.path.isfile(lxc):
+        print("Using " + lxc + " to generate man pages.")
+    else:
+        print("Cannot find lxc in " + lxc)
+        exit(2)
+else:
+    lxc = "../lxc.bin"
+
+# Generate man pages content
+
+os.makedirs('.sphinx/deps/manpages', exist_ok=True)
+subprocess.run([lxc, 'manpage', '.sphinx/deps/manpages/', '--format=md'],
+               check=True)
+
+# Preprocess man pages content
+
+for page in [x for x in os.listdir('.sphinx/deps/manpages')
+             if os.path.isfile(os.path.join('.sphinx/deps/manpages/', x))]:
+
+    # replace underscores with slashes to create a directory structure
+    pagepath = page.replace('_', '/')
+
+    # for each generated page, add an anchor, fix the title, and adjust the
+    # heading levels
+    with open(os.path.join('.sphinx/deps/manpages/', page), 'r') as mdfile:
+        content = mdfile.readlines()
+
+    os.makedirs(os.path.dirname(os.path.join('.sphinx/deps/manpages/', pagepath)),
+                exist_ok=True)
+
+    with open(os.path.join('.sphinx/deps/manpages/', pagepath), 'w') as mdfile:
+        mdfile.write('(' + page + ')=\n')
+        for line in content:
+            if line.startswith('###### Auto generated'):
+                continue
+            elif line.startswith('## '):
+                mdfile.write('# `' + line[3:].rstrip() + '`\n')
+            elif line.startswith('##'):
+                mdfile.write(line[1:])
+            else:
+                mdfile.write(line)
+
+    # remove the input page (unless the file path doesn't change)
+    if '_' in page:
+        os.remove(os.path.join('.sphinx/deps/manpages/', page))
+
+# Complete and copy man pages content
+
+for folder, subfolders, files in os.walk('.sphinx/deps/manpages'):
+
+    # for each subfolder, add toctrees to the parent page that
+    # include the subpages
+    for subfolder in subfolders:
+        with open(os.path.join(folder, subfolder + '.md'), 'a') as parent:
+            parent.write('```{toctree}\n:titlesonly:\n:glob:\n:hidden:\n\n' +
+                         subfolder + '/*\n```\n')
+
+    # for each file, if the content is different to what has been generated
+    # before, copy the file to the reference/manpages folder
+    # (copying all would mess up the incremental build)
+    for f in files:
+        sourcefile = os.path.join(folder, f)
+        targetfile = os.path.join('reference/manpages/',
+                                  os.path.relpath(folder,
+                                                  '.sphinx/deps/manpages'),
+                                  f)
+
+        if (not os.path.isfile(targetfile) or
+            not filecmp.cmp(sourcefile, targetfile, shallow=False)):
+
+            os.makedirs(os.path.dirname(targetfile), exist_ok=True)
+            os.system('cp ' + sourcefile + ' ' + targetfile)
+
+### End MAN PAGES ###
+
 # Project config.
 project = "LXD"
 author = "LXD contributors"
@@ -169,6 +253,7 @@
     'https://127.0.0.1:8443/1.0',
     'https://web.libera.chat/#lxd'
 ]
+linkcheck_exclude_documents = [r'.*/manpages/.*']
 
 # Setup redirects (https://documatt.gitlab.io/sphinx-reredirects/usage.html)
 redirects = {

doc/debugging.md

@@ -40,7 +40,7 @@ See the [RESTful API](rest-api.md) for available API.
 ## REST API through HTTPS
 
 {ref}`HTTPS connection to LXD <security>` requires valid
-client certificate that is generated on first `lxc remote add`. This
+client certificate that is generated on first [`lxc remote add`](lxc_remote_add.md). This
 certificate should be passed to connection tools for authentication
 and encryption.
 

doc/explanation/clustering.md

@@ -10,7 +10,7 @@ discourse: 15728
 
 To spread the total workload over several servers, LXD can be run in clustering mode.
 In this scenario, any number of LXD servers share the same distributed database that holds the configuration for the cluster members and their instances.
-The LXD cluster can be managed uniformly using the `lxc` client or the REST API.
+The LXD cluster can be managed uniformly using the [`lxc`](lxc.md) client or the REST API.
 
 This feature was introduced as part of the [`clustering`](../api-extensions.md#clustering) API extension and is available since LXD 3.0.
 
@@ -88,7 +88,7 @@ See {ref}`cluster-recover` for more information.
 You can use failure domains to indicate which cluster members should be given preference when assigning roles to a cluster member that has gone offline.
 For example, if a cluster member that currently has the database role gets shut down, LXD tries to assign its database role to another cluster member in the same failure domain, if one is available.
 
-To update the failure domain of a cluster member, use the `lxc cluster edit <member>` command and change the `failure_domain` property from `default` to another string.
+To update the failure domain of a cluster member, use the [`lxc cluster edit <member>`](lxc_cluster_edit.md) command and change the `failure_domain` property from `default` to another string.
 
 (clustering-member-config)=
 ### Member configuration

doc/explanation/instance_config.md

@@ -16,7 +16,7 @@ Instance options
 : Instance options are configuration options that are related directly to the instance.
   They include, for example, startup options, security settings, hardware limits, kernel modules, snapshots and user keys.
   These options can be specified as key/value pairs during instance creation (through the `--config key=value` flag).
-  After creation, they can be configured with the `lxc config set` and `lxc config unset` commands.
+  After creation, they can be configured with the [`lxc config set`](lxc_config_set.md) and [`lxc config unset`](lxc_config_unset.md) commands.
 
   In the YAML configuration, options are located under the `config` entry.
 
@@ -25,7 +25,7 @@ Instance options
 Instance devices
 : Instance devices are attached to an instance.
   They include, for example, network interfaces, mount points, USB and GPU devices.
-  Devices are usually added after an instance is created with the `lxc config device add` command, but they can also be added to a profile or a YAML configuration file that is used to create an instance.
+  Devices are usually added after an instance is created with the [`lxc config device add`](lxc_config_device_add.md) command, but they can also be added to a profile or a YAML configuration file that is used to create an instance.
 
   Each type of device has its own specific set of options, referred to as *instance device options*.
 

doc/explanation/lxd_lxc.md

@@ -46,7 +46,7 @@ LXD daemon
 LXD client
 : The `lxc` command is a command-line client for LXD, which you can use to interact with the LXD daemon.
   You use the `lxc` command to manage your instances, the server settings, and overall the entities you create in LXD.
-  See `lxc --help` for an overview of all available subcommands.
+  See [`lxc --help`](lxc.md) for an overview of all available subcommands.
 
   The `lxc` tool is not the only client you can use to interact with the LXD daemon.
   You can also use the API, the UI, or a custom LXD client.