From c0251c5e0ecad7c8dbd085c0d1a7326628dbd149 Mon Sep 17 00:00:00 2001
From: Henry <chocolatkey@gmail.com>
Date: Mon, 17 Jun 2024 00:32:30 -0700
Subject: [PATCH] Upgrades to rwp utility and repo CI/CD (#91)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Mickaël Menu <mickael.menu@gmail.com>
---
 .github/workflows/build.yml         |  10 +-
 .github/workflows/release.yml       |  51 +++++-
 .goreleaser.yaml                    |  26 ++-
 CHANGELOG.md                        |  11 +-
 Dockerfile                          |  56 ++++++
 README.md                           |   5 +
 cmd/rwp/cmd/serve.go                | 112 ++++++++++++
 cmd/rwp/cmd/serve/api.go            | 257 ++++++++++++++++++++++++++++
 cmd/rwp/cmd/serve/cache/local.go    |  90 ++++++++++
 cmd/rwp/cmd/serve/cache/pubcache.go |  22 +++
 cmd/rwp/cmd/serve/helpers.go        |  64 +++++++
 cmd/rwp/cmd/serve/router.go         |  47 +++++
 cmd/rwp/cmd/serve/server.go         |  32 ++++
 go.mod                              |  22 ++-
 go.sum                              |  68 ++++++--
 15 files changed, 819 insertions(+), 54 deletions(-)
 create mode 100644 Dockerfile
 create mode 100644 cmd/rwp/cmd/serve.go
 create mode 100644 cmd/rwp/cmd/serve/api.go
 create mode 100644 cmd/rwp/cmd/serve/cache/local.go
 create mode 100644 cmd/rwp/cmd/serve/cache/pubcache.go
 create mode 100644 cmd/rwp/cmd/serve/helpers.go
 create mode 100644 cmd/rwp/cmd/serve/router.go
 create mode 100644 cmd/rwp/cmd/serve/server.go

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ea303847..c26fd3ce 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -2,21 +2,21 @@ name: Build and Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, develop ]
   pull_request:
     branches: [ '**' ]
 
 jobs:
 
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, arm64]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.21
+        go-version: '>=1.22.0'
 
     - name: Build
       run: go build -v ./...
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d004e69e..de394ebb 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,24 +7,59 @@ on:
 
 permissions:
   contents: write
+  packages: write
+
+env:
+  IMAGE_NAME: rwp
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, arm64]
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          go-version: '>=1.21.0'
-          cache: true
-      - uses: goreleaser/goreleaser-action@v4
+          go-version: '>=1.22.0'
+      - name: Build release
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
           version: latest
           args: release --clean
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  container:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push Docker image
+        run: docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker buildx build --push \
+            --tag $IMAGE_ID:$VERSION \
+            --platform linux/amd64,linux/arm64,linux/arm/v7 .
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index 7ef8200d..892bcd44 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -2,29 +2,37 @@
 before:
   hooks:
     - go mod tidy
+    - go generate ./...
 builds:
   - main: ./cmd/rwp/
+    env:
+      - CGO_ENABLED=0
     id: rwp
     binary: rwp
     goos:
       - linux
       - windows
       - darwin
+    goamd64:
+      - v3
 
-  - main: ./cmd/server/
-    id: rwp-server
-    binary: rwp-server
-    goos:
-      - linux
-      - windows
-      - darwin
+#  - main: ./cmd/server/
+#    env:
+#      - CGO_ENABLED=0
+#    id: rwp-server
+#    binary: rwp-server
+#    goos:
+#      - linux
+#      - windows
+#      - darwin
 
 archives:
   - format: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
+    # Used to start with {{ .ProjectName }}
     name_template: >-
-      {{ .ProjectName }}_
-      {{- title .Os }}_
+      rwp_
+      {{- tolower .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5563e3bd..d04ed03e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,13 +6,4 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
-### Added
-
-* Add the [Media Type API](https://readium.org/architecture/proposals/001-media-type.html).
-* Add the file and archive fetchers of the [Fetcher API](https://readium.org/architecture/proposals/002-composite-fetcher-api.html).
-
-### Changed
-
-* Restructuring of the repo's folders
-* Removal of legacy models (LCP etc.)
-* Updated shared models to latest specs
\ No newline at end of file
+TODO
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..421c7392
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,56 @@
+FROM --platform=$BUILDPLATFORM golang:1-bookworm as builder
+ARG BUILDARCH TARGETOS TARGETARCH
+
+# Install GoReleaser
+RUN wget --no-verbose "https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/goreleaser_1.26.2_$BUILDARCH.deb"
+RUN dpkg -i "goreleaser_1.26.2_$BUILDARCH.deb"
+
+# Create and change to the app directory.
+WORKDIR /app
+
+# Retrieve application dependencies.
+# This allows the container build to reuse cached dependencies.
+# Expecting to copy go.mod and if present go.sum.
+COPY go.* ./
+RUN go mod download
+
+# Copy local code to the container image.
+COPY . ./
+
+# RUN git lfs pull && ls -alh publications
+
+# Run goreleaser
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    GOOS=$TARGETOS GOARCH=$TARGETARCH goreleaser build --single-target --id rwp --skip=validate --snapshot --output ./rwp
+
+# Run tests
+# FROM builder AS tester
+# RUN go test ./...
+
+# Produces very small images
+FROM gcr.io/distroless/static-debian12 AS packager
+
+# Extra metadata
+LABEL org.opencontainers.image.source="https://github.com/readium/go-toolkit"
+
+# Add Fedora's mimetypes (pretty up-to-date and expansive)
+# since the distroless container doesn't have any. Go uses
+# this file as part of its mime package, and readium/go-toolkit
+# has a mediatype package that falls back to Go's mime
+# package to discover a file's mimetype when all else fails.
+ADD https://pagure.io/mailcap/raw/master/f/mime.types /etc/
+
+# Add two demo EPUBs to the container by default
+ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/moby-dick.epub /srv/publications/
+ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/BellaOriginal3.epub /srv/publications/
+
+# Copy built Go binary
+COPY --from=builder "/app/rwp" /opt/
+
+EXPOSE 15080
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/opt/rwp"]
+CMD ["serve", "/srv/publications", "--address", "0.0.0.0"]
\ No newline at end of file
diff --git a/README.md b/README.md
index ed7efc10..2bddc6bf 100644
--- a/README.md
+++ b/README.md
@@ -56,3 +56,8 @@ rwp manifest --infer-a11y=merged publication.epub  | jq .metadata
 | `feature` | `tableOfContents` | If the publications contains a table of contents (check for the presence of a `toc` collection in RWPM) |
 | `feature` | `MathML` | If the publication contains any resource with MathML (check for the presence of the `contains` property where the value is `mathml` in `readingOrder` or `resources` in RWPM) |
 | `feature` | `synchronizedAudioText` | If the publication contains any reference to Media Overlays (TBD in RWPM) |
+
+### HTTP streaming of local publications
+
+`rwp serve` starts an HTTP server that serves EPUB, CBZ and other compatible formats from a given directory.
+A log is printed to stdout.
\ No newline at end of file
diff --git a/cmd/rwp/cmd/serve.go b/cmd/rwp/cmd/serve.go
new file mode 100644
index 00000000..5e4bd192
--- /dev/null
+++ b/cmd/rwp/cmd/serve.go
@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"log/slog"
+
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve"
+	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/spf13/cobra"
+)
+
+var debugFlag bool
+
+var bindAddressFlag string
+
+var bindPortFlag uint16
+
+var serveCmd = &cobra.Command{
+	Use:   "serve <directory>",
+	Short: "Start a local HTTP server, serving a specified directory of publications",
+	Long: `Start a local HTTP server, serving a specified directory of publications.
+
+This command will start an HTTP serve listening by default on 'localhost:15080',
+serving all compatible files (EPUB, PDF, CBZ, etc.) found in the directory
+as Readium Web Publications. To get started, the manifest can be accessed from
+'http://localhost:15080/<filename in base64url encoding without padding>/manifest.json'.
+This file serves as the entry point and contains metadata and links to the rest
+of the files that can be accessed for the publication.
+
+For debugging purposes, the server also exposes a '/list.json' endpoint that
+returns a list of all the publications found in the directory along with their
+encoded paths. This will be replaced by an OPDS 2 feed in a future release.
+
+Note: This server is not meant for production usage, and should not be exposed
+to the internet except for testing/debugging purposes.`,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if len(args) == 0 {
+			return errors.New("expects a directory path to serve publications from")
+		} else if len(args) > 1 {
+			return errors.New("accepts a directory path")
+		}
+		return nil
+	},
+
+	SuggestFor: []string{"server"},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// By the time we reach this point, we know that the arguments were
+		// properly parsed, and we don't want to show the usage if an API error
+		// occurs.
+		cmd.SilenceUsage = true
+
+		path := filepath.Clean(args[0])
+		fi, err := os.Stat(path)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return fmt.Errorf("given directory %s does not exist", path)
+			}
+			return fmt.Errorf("failed to stat %s: %w", path, err)
+		}
+		if !fi.IsDir() {
+			return fmt.Errorf("given path %s is not a directory", path)
+		}
+
+		// Log level
+		if debugFlag {
+			slog.SetLogLoggerLevel(slog.LevelDebug)
+		} else {
+			slog.SetLogLoggerLevel(slog.LevelInfo)
+		}
+
+		pubServer := serve.NewServer(serve.ServerConfig{
+			Debug:             debugFlag,
+			BaseDirectory:     path,
+			JSONIndent:        indentFlag,
+			InferA11yMetadata: streamer.InferA11yMetadata(inferA11yFlag),
+		})
+
+		bind := fmt.Sprintf("%s:%d", bindAddressFlag, bindPortFlag)
+		httpServer := &http.Server{
+			ReadTimeout:    10 * time.Second,
+			WriteTimeout:   10 * time.Second,
+			MaxHeaderBytes: 1 << 20,
+			Addr:           bind,
+			Handler:        pubServer.Routes(),
+		}
+		slog.Info("Starting HTTP server", "address", "http://"+httpServer.Addr)
+		if err := httpServer.ListenAndServe(); err != http.ErrServerClosed {
+			slog.Error("Server stopped", "error", err)
+		} else {
+			slog.Info("Goodbye!")
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(serveCmd)
+
+	serveCmd.Flags().StringVarP(&bindAddressFlag, "address", "a", "localhost", "Address to bind the HTTP server to")
+	serveCmd.Flags().Uint16VarP(&bindPortFlag, "port", "p", 15080, "Port to bind the HTTP server to")
+	serveCmd.Flags().StringVarP(&indentFlag, "indent", "i", "", "Indentation used to pretty-print JSON files")
+	serveCmd.Flags().Var(&inferA11yFlag, "infer-a11y", "Infer accessibility metadata: no, merged, split")
+	serveCmd.Flags().BoolVarP(&debugFlag, "debug", "d", false, "Enable debug mode")
+
+}
diff --git a/cmd/rwp/cmd/serve/api.go b/cmd/rwp/cmd/serve/api.go
new file mode 100644
index 00000000..552b6fbc
--- /dev/null
+++ b/cmd/rwp/cmd/serve/api.go
@@ -0,0 +1,257 @@
+package serve
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/gorilla/mux"
+	httprange "github.com/gotd/contrib/http_range"
+	"github.com/pkg/errors"
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache"
+	"github.com/readium/go-toolkit/pkg/asset"
+	"github.com/readium/go-toolkit/pkg/manifest"
+	"github.com/readium/go-toolkit/pkg/pub"
+	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/zeebo/xxh3"
+)
+
+type demoListItem struct {
+	Filename string `json:"filename"`
+	Path     string `json:"path"`
+}
+
+func (s *Server) demoList(w http.ResponseWriter, req *http.Request) {
+	fi, err := os.ReadDir(s.config.BaseDirectory)
+	if err != nil {
+		slog.Error("failed reading publications directory", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+	files := make([]demoListItem, len(fi))
+	for i, f := range fi {
+		files[i] = demoListItem{
+			Filename: f.Name(),
+			Path:     base64.RawURLEncoding.EncodeToString([]byte(f.Name())),
+		}
+	}
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", s.config.JSONIndent)
+	enc.Encode(files)
+}
+
+func (s *Server) getPublication(filename string) (*pub.Publication, error) {
+	fpath, err := base64.RawURLEncoding.DecodeString(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	cp := filepath.Clean(string(fpath))
+	dat, ok := s.lfu.Get(cp)
+	if !ok {
+		pub, err := streamer.New(streamer.Config{
+			InferA11yMetadata: s.config.InferA11yMetadata,
+		}).Open(asset.File(filepath.Join(s.config.BaseDirectory, cp)), "")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed opening "+cp)
+		}
+
+		// TODO: Remove this after we make links relative in the go-toolkit
+		for i, link := range pub.Manifest.Links {
+			pub.Manifest.Links[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.Resources {
+			pub.Manifest.Resources[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.ReadingOrder {
+			pub.Manifest.ReadingOrder[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.TableOfContents {
+			pub.Manifest.TableOfContents[i] = makeRelative(link)
+		}
+		var makeCollectionRelative func(mp manifest.PublicationCollectionMap)
+		makeCollectionRelative = func(mp manifest.PublicationCollectionMap) {
+			for i := range mp {
+				for j := range mp[i] {
+					for k := range mp[i][j].Links {
+						mp[i][j].Links[k] = makeRelative(mp[i][j].Links[k])
+					}
+					makeCollectionRelative(mp[i][j].Subcollections)
+				}
+			}
+		}
+		makeCollectionRelative(pub.Manifest.Subcollections)
+
+		// Cache the publication
+		encPub := &cache.CachedPublication{Publication: pub}
+		s.lfu.Set(cp, encPub)
+
+		return encPub.Publication, nil
+	}
+	return dat.(*cache.CachedPublication).Publication, nil
+}
+
+func (s *Server) getManifest(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	filename := vars["path"]
+
+	// Load the publication
+	publication, err := s.getPublication(filename)
+	if err != nil {
+		slog.Error("failed opening publication", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Create "self" link in manifest
+	scheme := "http://"
+	if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" {
+		// Note: this is never going to be 100% accurate behind proxies,
+		// but it's better than nothing for a dev server.
+		scheme = "https://"
+	}
+	rPath, _ := s.router.Get("manifest").URLPath("path", vars["path"])
+	selfLink := &manifest.Link{
+		Rels: manifest.Strings{"self"},
+		Type: conformsToAsMimetype(publication.Manifest.Metadata.ConformsTo),
+		Href: scheme + req.Host + rPath.String(),
+	}
+
+	// Marshal the manifest
+	j, err := json.Marshal(publication.Manifest.ToMap(selfLink))
+	if err != nil {
+		slog.Error("failed marshalling manifest JSON", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Indent JSON
+	var identJSON bytes.Buffer
+	if s.config.JSONIndent == "" {
+		_, err = identJSON.Write(j)
+		if err != nil {
+			slog.Error("failed writing manifest JSON to buffer", "error", err)
+			w.WriteHeader(500)
+			return
+		}
+	} else {
+		err = json.Indent(&identJSON, j, "", s.config.JSONIndent)
+		if err != nil {
+			slog.Error("failed indenting manifest JSON", "error", err)
+			w.WriteHeader(500)
+			return
+		}
+	}
+
+	// Add headers
+	w.Header().Set("content-type", conformsToAsMimetype(publication.Manifest.Metadata.ConformsTo)+"; charset=utf-8")
+	w.Header().Set("cache-control", "private, must-revalidate")
+	w.Header().Set("access-control-allow-origin", "*") // TODO: provide options?
+
+	// Etag based on hash of the manifest bytes
+	etag := `"` + strconv.FormatUint(xxh3.Hash(identJSON.Bytes()), 36) + `"`
+	w.Header().Set("Etag", etag)
+	if match := req.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	// Write response body
+	_, err = identJSON.WriteTo(w)
+	if err != nil {
+		slog.Error("failed writing manifest JSON to response writer", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+}
+
+func (s *Server) getAsset(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	filename := vars["path"]
+
+	// Load the publication
+	publication, err := s.getPublication(filename)
+	if err != nil {
+		slog.Error("failed opening publication", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Make sure the asset exists in the publication
+	href := path.Clean(vars["asset"])
+	link := publication.Find(href)
+	if link == nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	// Get the asset from the publication
+	res := publication.Get(*link)
+	defer res.Close()
+
+	// Get asset length in bytes
+	l, rerr := res.Length()
+	if rerr != nil {
+		w.WriteHeader(rerr.HTTPStatus())
+		w.Write([]byte(rerr.Error()))
+		return
+	}
+
+	// Patch mimetype where necessary
+	contentType := link.MediaType().String()
+	if sub, ok := mimeSubstitutions[contentType]; ok {
+		contentType = sub
+	}
+	w.Header().Set("content-type", contentType)
+	w.Header().Set("cache-control", "private, max-age=86400, immutable")
+	w.Header().Set("content-length", strconv.FormatInt(l, 10))
+	w.Header().Set("access-control-allow-origin", "*") // TODO: provide options?
+
+	var start, end int64
+	// Range reading assets
+	rangeHeader := r.Header.Get("range")
+	if rangeHeader != "" {
+		rng, err := httprange.ParseRange(rangeHeader, l)
+		if err != nil {
+			slog.Error("failed parsing range header", "error", err)
+			w.WriteHeader(http.StatusLengthRequired)
+			return
+		}
+		if len(rng) > 1 {
+			slog.Error("no support for multiple read ranges")
+			w.WriteHeader(http.StatusNotImplemented)
+			return
+		}
+		if len(rng) > 0 {
+			w.Header().Set("content-range", rng[0].ContentRange(l))
+			start = rng[0].Start
+			end = start + rng[0].Length - 1
+			w.Header().Set("content-length", strconv.FormatInt(rng[0].Length, 10))
+		}
+	}
+	if w.Header().Get("content-range") != "" {
+		w.WriteHeader(http.StatusPartialContent)
+	}
+
+	// Stream the asset
+	_, rerr = res.Stream(w, start, end)
+	if rerr != nil {
+		if errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET) {
+			// Ignore client errors
+			return
+		}
+
+		slog.Error("failed streaming asset", "error", rerr.Error())
+	}
+
+}
diff --git a/cmd/rwp/cmd/serve/cache/local.go b/cmd/rwp/cmd/serve/cache/local.go
new file mode 100644
index 00000000..0ddb8899
--- /dev/null
+++ b/cmd/rwp/cmd/serve/cache/local.go
@@ -0,0 +1,90 @@
+package cache
+
+// Originally from https://github.com/go-redis/cache/blob/v8.4.3/local.go
+// Modified to store interface{} instead of []byte
+
+import (
+	"sync"
+	"time"
+
+	"github.com/vmihailenco/go-tinylfu"
+	"golang.org/x/exp/rand"
+)
+
+type Evictable interface {
+	OnEvict()
+}
+
+type LocalCache interface {
+	Set(key string, data Evictable)
+	Get(key string) (Evictable, bool)
+	Del(key string)
+}
+
+type TinyLFU struct {
+	mu     sync.Mutex
+	rand   *rand.Rand
+	lfu    *tinylfu.T
+	ttl    time.Duration
+	offset time.Duration
+}
+
+var _ LocalCache = (*TinyLFU)(nil)
+
+func NewTinyLFU(size int, ttl time.Duration) *TinyLFU {
+	const maxOffset = 10 * time.Second
+
+	offset := ttl / 10
+	if offset > maxOffset {
+		offset = maxOffset
+	}
+
+	return &TinyLFU{
+		rand:   rand.New(rand.NewSource(uint64(time.Now().UnixNano()))),
+		lfu:    tinylfu.New(size, 100000),
+		ttl:    ttl,
+		offset: offset,
+	}
+}
+
+func (c *TinyLFU) UseRandomizedTTL(offset time.Duration) {
+	c.offset = offset
+}
+
+func (c *TinyLFU) Set(key string, b Evictable) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	ttl := c.ttl
+	if c.offset > 0 {
+		ttl += time.Duration(c.rand.Int63n(int64(c.offset)))
+	}
+
+	c.lfu.Set(&tinylfu.Item{
+		Key:      key,
+		Value:    b,
+		ExpireAt: time.Now().Add(ttl),
+		OnEvict: func() {
+			b.OnEvict()
+		},
+	})
+}
+
+func (c *TinyLFU) Get(key string) (Evictable, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	val, ok := c.lfu.Get(key)
+	if !ok {
+		return nil, false
+	}
+
+	return val.(Evictable), true
+}
+
+func (c *TinyLFU) Del(key string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.lfu.Del(key)
+}
diff --git a/cmd/rwp/cmd/serve/cache/pubcache.go b/cmd/rwp/cmd/serve/cache/pubcache.go
new file mode 100644
index 00000000..52a63f5a
--- /dev/null
+++ b/cmd/rwp/cmd/serve/cache/pubcache.go
@@ -0,0 +1,22 @@
+package cache
+
+import (
+	"github.com/readium/go-toolkit/pkg/pub"
+)
+
+// CachedPublication implements Evictable
+type CachedPublication struct {
+	*pub.Publication
+}
+
+func EncapsulatePublication(pub *pub.Publication) *CachedPublication {
+	cp := &CachedPublication{pub}
+	return cp
+}
+
+func (cp *CachedPublication) OnEvict() {
+	// Cleanup
+	if cp.Publication != nil {
+		cp.Publication.Close()
+	}
+}
diff --git a/cmd/rwp/cmd/serve/helpers.go b/cmd/rwp/cmd/serve/helpers.go
new file mode 100644
index 00000000..6a25921a
--- /dev/null
+++ b/cmd/rwp/cmd/serve/helpers.go
@@ -0,0 +1,64 @@
+package serve
+
+import (
+	"strings"
+
+	"github.com/readium/go-toolkit/pkg/manifest"
+)
+
+var mimeSubstitutions = map[string]string{
+	"application/vnd.ms-opentype":          "font/otf",                                            // Not just because it's sane, but because CF will compress it!
+	"application/vnd.readium.content+json": "application/vnd.readium.content+json; charset=utf-8", // Need utf-8 encoding
+}
+
+var compressableMimes = []string{
+	"application/javascript",
+	"application/x-javascript",
+	"image/x-icon",
+	"text/css",
+	"text/html",
+	"application/xhtml+xml",
+	"application/webpub+json",
+	"application/divina+json",
+	"application/vnd.readium.position-list+json",
+	"application/vnd.readium.content+json",
+	"application/audiobook+json",
+	"font/ttf",
+	"application/ttf",
+	"application/x-ttf",
+	"application/x-font-ttf",
+	"font/otf",
+	"application/otf",
+	"application/x-otf",
+	"application/vnd.ms-opentype",
+	"font/opentype",
+	"application/opentype",
+	"application/x-opentype",
+	"application/truetype",
+	"application/font-woff",
+	"font/x-woff",
+	"application/vnd.ms-fontobject",
+}
+
+func makeRelative(link manifest.Link) manifest.Link {
+	link.Href = strings.TrimPrefix(link.Href, "/")
+	for i, alt := range link.Alternates {
+		link.Alternates[i].Href = strings.TrimPrefix(alt.Href, "/")
+	}
+	return link
+}
+
+func conformsToAsMimetype(conformsTo manifest.Profiles) string {
+	mime := "application/webpub+json"
+	for _, profile := range conformsTo {
+		if profile == manifest.ProfileDivina {
+			mime = "application/divina+json"
+		} else if profile == manifest.ProfileAudiobook {
+			mime = "application/audiobook+json"
+		} else {
+			continue
+		}
+		break
+	}
+	return mime
+}
diff --git a/cmd/rwp/cmd/serve/router.go b/cmd/rwp/cmd/serve/router.go
new file mode 100644
index 00000000..e3b5fd19
--- /dev/null
+++ b/cmd/rwp/cmd/serve/router.go
@@ -0,0 +1,47 @@
+package serve
+
+import (
+	"net/http"
+	"net/http/pprof"
+
+	"github.com/CAFxX/httpcompression"
+	"github.com/gorilla/mux"
+)
+
+func (s *Server) Routes() *mux.Router {
+	r := mux.NewRouter()
+
+	r.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+
+	if s.config.Debug {
+		r.HandleFunc("/debug/pprof/", pprof.Index)
+		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+		r.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
+		r.Handle("/debug/pprof/block", pprof.Handler("block"))
+		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		r.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
+		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+	}
+
+	r.HandleFunc("/list.json", s.demoList).Name("demo_list")
+
+	pub := r.PathPrefix("/{path}").Subrouter()
+	// TODO: publication loading middleware with pub.Use()
+	pub.Use(func(h http.Handler) http.Handler {
+		adapter, _ := httpcompression.DefaultAdapter(httpcompression.ContentTypes(compressableMimes, false))
+		return adapter(h)
+	})
+	pub.HandleFunc("/manifest.json", s.getManifest).Name("manifest")
+	pub.HandleFunc("/{asset:.*}", s.getAsset).Name("asset")
+
+	s.router = r
+	return r
+}
diff --git a/cmd/rwp/cmd/serve/server.go b/cmd/rwp/cmd/serve/server.go
new file mode 100644
index 00000000..20f1b5df
--- /dev/null
+++ b/cmd/rwp/cmd/serve/server.go
@@ -0,0 +1,32 @@
+package serve
+
+import (
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache"
+	"github.com/readium/go-toolkit/pkg/streamer"
+)
+
+type ServerConfig struct {
+	Debug             bool
+	BaseDirectory     string
+	JSONIndent        string
+	InferA11yMetadata streamer.InferA11yMetadata
+}
+
+type Server struct {
+	config ServerConfig
+	router *mux.Router
+	lfu    *cache.TinyLFU
+}
+
+const MaxCachedPublicationAmount = 10
+const MaxCachedPublicationTTL = time.Second * time.Duration(600)
+
+func NewServer(config ServerConfig) *Server {
+	return &Server{
+		config: config,
+		lfu:    cache.NewTinyLFU(MaxCachedPublicationAmount, MaxCachedPublicationTTL),
+	}
+}
diff --git a/go.mod b/go.mod
index a8797d41..660112ce 100644
--- a/go.mod
+++ b/go.mod
@@ -3,10 +3,12 @@ module github.com/readium/go-toolkit
 go 1.21
 
 require (
+	github.com/CAFxX/httpcompression v0.0.9
 	github.com/agext/regexp v1.3.0
 	github.com/andybalholm/cascadia v1.3.2
 	github.com/deckarep/golang-set v1.7.1
 	github.com/gorilla/mux v1.7.4
+	github.com/gotd/contrib v0.20.0
 	github.com/opds-community/libopds2-go v0.0.0-20170628075933-9c163cf60f6e
 	github.com/pdfcpu/pdfcpu v0.5.0
 	github.com/pkg/errors v0.9.1
@@ -16,25 +18,31 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.9.0
 	github.com/trimmer-io/go-xmp v1.0.0
 	github.com/urfave/negroni v1.0.0
-	golang.org/x/net v0.10.0
-	golang.org/x/text v0.12.0
+	github.com/vmihailenco/go-tinylfu v0.2.2
+	github.com/zeebo/xxh3 v1.0.2
+	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10
+	golang.org/x/net v0.23.0
+	golang.org/x/text v0.14.0
 )
 
 require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antchfx/xpath v1.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hhrutter/lzw v1.0.0 // indirect
 	github.com/hhrutter/tiff v1.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
@@ -42,8 +50,8 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	golang.org/x/image v0.11.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 3e78ec78..345a44cc 100644
--- a/go.sum
+++ b/go.sum
@@ -39,8 +39,12 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/CAFxX/httpcompression v0.0.9 h1:0ue2X8dOLEpxTm8tt+OdHcgA+gbDge0OqFQWGKSqgrg=
+github.com/CAFxX/httpcompression v0.0.9/go.mod h1:XX8oPZA+4IDcfZ0A71Hz0mZsv/YJOgYygkFhizVPilM=
 github.com/agext/regexp v1.3.0 h1:6+9tp+S41TU48gFNV47bX+pp1q7WahGofw6JccmsCDs=
 github.com/agext/regexp v1.3.0/go.mod h1:6phv1gViOJXWcTfpxOi9VMS+MaSAo+SUDf7do3ur1HA=
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8=
@@ -52,6 +56,9 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -114,6 +121,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f h1:jopqB+UTSdJGEJT8tEqYyE29zN91fi2827oLET8tl7k=
+github.com/google/brotli/go/cbrotli v0.0.0-20230829110029-ed738e842d2f/go.mod h1:nOPhAkwVliJdNTkj3gXpljmWhjc4wCaVqbMJcPKWP4s=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -147,10 +156,10 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw=
-github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gotd/contrib v0.20.0 h1:1Wc4+HMQiIKYQuGHVwVksIx152HFTP6B5n88dDe0ZYw=
+github.com/gotd/contrib v0.20.0/go.mod h1:P6o8W4niqhDPHLA0U+SA/L7l3BQHYLULpeHfRSePn9o=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
@@ -184,16 +193,23 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
+github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -206,8 +222,9 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -218,6 +235,8 @@ github.com/pdfcpu/pdfcpu v0.5.0 h1:F3wC4bwPbaJM+RPgm1D0Q4SAUwxElw7BhwNvL3iPgDo=
 github.com/pdfcpu/pdfcpu v0.5.0/go.mod h1:UPcHdWcMw1V6Bo5tcWHd3jZfkG8cwUwrJkQOlB6o+7g=
 github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -232,14 +251,14 @@ github.com/relvacode/iso8601 v1.1.0 h1:2nV8sp0eOjpoKQ2vD3xSDygsjAx37NHG2UlZiCkDH
 github.com/relvacode/iso8601 v1.1.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -254,25 +273,40 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/trimmer-io/go-xmp v1.0.0 h1:zY8bolSga5kOjBAaHS6hrdxLgEoYuT875xTy0QDwZWs=
 github.com/trimmer-io/go-xmp v1.0.0/go.mod h1:Aaptr9sp1lLv7UnCAdQ+gSHZyY2miYaKmcNVj7HRBwA=
+github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
+github.com/valyala/gozstd v1.20.1 h1:xPnnnvjmaDDitMFfDxmQ4vpx0+3CdTg2o3lALvXTU/g=
+github.com/valyala/gozstd v1.20.1/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
+github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI=
+github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -304,6 +338,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck=
+golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
@@ -373,8 +409,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -448,8 +484,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -464,8 +500,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -632,8 +669,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=