Upgrades to rwp utility and repo CI/CD

.github/workflows/build.yml

@@ -2,21 +2,21 @@ name: Build and Test
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, develop ]
   pull_request:
     branches: [ '**' ]
 
 jobs:
 
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, arm64]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.21
+        go-version: '>=1.22.0'
 
     - name: Build
       run: go build -v ./...

.github/workflows/release.yml

@@ -7,24 +7,59 @@ on:
 
 permissions:
   contents: write
+  packages: write
+
+env:
+  IMAGE_NAME: rwp
 
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, arm64]
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
-      - uses: actions/setup-go@v3
+      - name: Set up Go
+        uses: actions/setup-go@v5
         with:
-          go-version: '>=1.21.0'
-          cache: true
-      - uses: goreleaser/goreleaser-action@v4
+          go-version: '>=1.22.0'
+      - name: Build release
+        uses: goreleaser/goreleaser-action@v5
         with:
           distribution: goreleaser
           version: latest
           args: release --clean
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  container:
+    runs-on: [self-hosted, arm64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push Docker image
+        run: docker buildx build --platform=linux/amd64,linux/arm64,linux/arm/v7 . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}"
+      - name: Log in to registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+          # This changes all uppercase characters to lowercase.
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+          # This strips the git ref prefix from the version.
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          # This strips the "v" prefix from the tag name.
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+          # This uses the Docker `latest` tag convention.
+          [ "$VERSION" == "main" ] && VERSION=latest
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+          docker buildx build --push \
+            --tag $IMAGE_ID:$VERSION \
+            --platform linux/amd64,linux/arm64,linux/arm/v7 .

.goreleaser.yaml

@@ -2,29 +2,37 @@
 before:
   hooks:
     - go mod tidy
+    - go generate ./...
 builds:
   - main: ./cmd/rwp/
+    env:
+      - CGO_ENABLED=0
     id: rwp
     binary: rwp
     goos:
       - linux
       - windows
       - darwin
+    goamd64:
+      - v3
 
-  - main: ./cmd/server/
-    id: rwp-server
-    binary: rwp-server
-    goos:
-      - linux
-      - windows
-      - darwin
+#  - main: ./cmd/server/
+#    env:
+#      - CGO_ENABLED=0
+#    id: rwp-server
+#    binary: rwp-server
+#    goos:
+#      - linux
+#      - windows
+#      - darwin
 
 archives:
   - format: tar.gz
     # this name template makes the OS and Arch compatible with the results of uname.
+    # Used to start with {{ .ProjectName }}
     name_template: >-
-      {{ .ProjectName }}_
-      {{- title .Os }}_
+      rwp_
+      {{- tolower .Os }}_
       {{- if eq .Arch "amd64" }}x86_64
       {{- else if eq .Arch "386" }}i386
       {{- else }}{{ .Arch }}{{ end }}

CHANGELOG.md

@@ -6,13 +6,4 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
-### Added
-
-* Add the [Media Type API](https://readium.org/architecture/proposals/001-media-type.html).
-* Add the file and archive fetchers of the [Fetcher API](https://readium.org/architecture/proposals/002-composite-fetcher-api.html).
-
-### Changed
-
-* Restructuring of the repo's folders
-* Removal of legacy models (LCP etc.)
-* Updated shared models to latest specs
+TODO

Dockerfile

@@ -0,0 +1,56 @@
+FROM --platform=$BUILDPLATFORM golang:1-bookworm as builder
+ARG BUILDARCH TARGETOS TARGETARCH
+
+# Install GoReleaser
+RUN wget --no-verbose "https://github.com/goreleaser/goreleaser/releases/download/v1.26.2/goreleaser_1.26.2_$BUILDARCH.deb"
+RUN dpkg -i "goreleaser_1.26.2_$BUILDARCH.deb"
+
+# Create and change to the app directory.
+WORKDIR /app
+
+# Retrieve application dependencies.
+# This allows the container build to reuse cached dependencies.
+# Expecting to copy go.mod and if present go.sum.
+COPY go.* ./
+RUN go mod download
+
+# Copy local code to the container image.
+COPY . ./
+
+# RUN git lfs pull && ls -alh publications
+
+# Run goreleaser
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg \
+    GOOS=$TARGETOS GOARCH=$TARGETARCH goreleaser build --single-target --id rwp --skip=validate --snapshot --output ./rwp
+
+# Run tests
+# FROM builder AS tester
+# RUN go test ./...
+
+# Produces very small images
+FROM gcr.io/distroless/static-debian12 AS packager
+
+# Extra metadata
+LABEL org.opencontainers.image.source="https://github.com/readium/go-toolkit"
+
+# Add Fedora's mimetypes (pretty up-to-date and expansive)
+# since the distroless container doesn't have any. Go uses
+# this file as part of its mime package, and readium/go-toolkit
+# has a mediatype package that falls back to Go's mime
+# package to discover a file's mimetype when all else fails.
+ADD https://pagure.io/mailcap/raw/master/f/mime.types /etc/
+
+# Add two demo EPUBs to the container by default
+ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/moby-dick.epub /srv/publications/
+ADD --chown=nonroot:nonroot https://readium-playground-files.storage.googleapis.com/demo/BellaOriginal3.epub /srv/publications/
+
+# Copy built Go binary
+COPY --from=builder "/app/rwp" /opt/
+
+EXPOSE 15080
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/opt/rwp"]
+CMD ["serve", "/srv/publications", "--address", "0.0.0.0"]

README.md

@@ -56,3 +56,8 @@ rwp manifest --infer-a11y=merged publication.epub  | jq .metadata
 | `feature` | `tableOfContents` | If the publications contains a table of contents (check for the presence of a `toc` collection in RWPM) |
 | `feature` | `MathML` | If the publication contains any resource with MathML (check for the presence of the `contains` property where the value is `mathml` in `readingOrder` or `resources` in RWPM) |
 | `feature` | `synchronizedAudioText` | If the publication contains any reference to Media Overlays (TBD in RWPM) |
+
+### HTTP streaming of local publications
+
+`rwp serve` starts an HTTP server that serves EPUB, CBZ and other compatible formats from a given directory.
+A log is printed to stdout.

cmd/rwp/cmd/serve.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"log/slog"
+
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve"
+	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/spf13/cobra"
+)
+
+var debugFlag bool
+
+var bindAddressFlag string
+
+var bindPortFlag uint16
+
+var serveCmd = &cobra.Command{
+	Use:   "serve <directory>",
+	Short: "Start a local HTTP server, serving a specified directory of publications",
+	Long: `Start a local HTTP server, serving a specified directory of publications.
+
+This command will start an HTTP serve listening by default on 'localhost:15080',
+serving all compatible files (EPUB, PDF, CBZ, etc.) found in the directory
+as Readium Web Publications. To get started, the manifest can be accessed from
+'http://localhost:15080/<filename in base64url encoding without padding>/manifest.json'.
+This file serves as the entry point and contains metadata and links to the rest
+of the files that can be accessed for the publication.
+
+For debugging purposes, the server also exposes a '/list.json' endpoint that
+returns a list of all the publications found in the directory along with their
+encoded paths. This will be replaced by an OPDS 2 feed in a future release.
+
+Note: This server is not meant for production usage, and should not be exposed
+to the internet except for testing/debugging purposes.`,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if len(args) == 0 {
+			return errors.New("expects a directory path to serve publications from")
+		} else if len(args) > 1 {
+			return errors.New("accepts a directory path")
+		}
+		return nil
+	},
+
+	SuggestFor: []string{"server"},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// By the time we reach this point, we know that the arguments were
+		// properly parsed, and we don't want to show the usage if an API error
+		// occurs.
+		cmd.SilenceUsage = true
+
+		path := filepath.Clean(args[0])
+		fi, err := os.Stat(path)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return fmt.Errorf("given directory %s does not exist", path)
+			}
+			return fmt.Errorf("failed to stat %s: %w", path, err)
+		}
+		if !fi.IsDir() {
+			return fmt.Errorf("given path %s is not a directory", path)
+		}
+
+		// Log level
+		if debugFlag {
+			slog.SetLogLoggerLevel(slog.LevelDebug)
+		} else {
+			slog.SetLogLoggerLevel(slog.LevelInfo)
+		}
+
+		pubServer := serve.NewServer(serve.ServerConfig{
+			Debug:             debugFlag,
+			BaseDirectory:     path,
+			JSONIndent:        indentFlag,
+			InferA11yMetadata: streamer.InferA11yMetadata(inferA11yFlag),
+		})
+
+		bind := fmt.Sprintf("%s:%d", bindAddressFlag, bindPortFlag)
+		httpServer := &http.Server{
+			ReadTimeout:    10 * time.Second,
+			WriteTimeout:   10 * time.Second,
+			MaxHeaderBytes: 1 << 20,
+			Addr:           bind,
+			Handler:        pubServer.Routes(),
+		}
+		slog.Info("Starting HTTP server", "address", "http://"+httpServer.Addr)
+		if err := httpServer.ListenAndServe(); err != http.ErrServerClosed {
+			slog.Error("Server stopped", "error", err)
+		} else {
+			slog.Info("Goodbye!")
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(serveCmd)
+
+	serveCmd.Flags().StringVarP(&bindAddressFlag, "address", "a", "localhost", "Address to bind the HTTP server to")
+	serveCmd.Flags().Uint16VarP(&bindPortFlag, "port", "p", 15080, "Port to bind the HTTP server to")
+	serveCmd.Flags().StringVarP(&indentFlag, "indent", "i", "", "Indentation used to pretty-print JSON files")
+	serveCmd.Flags().Var(&inferA11yFlag, "infer-a11y", "Infer accessibility metadata: no, merged, split")
+	serveCmd.Flags().BoolVarP(&debugFlag, "debug", "d", false, "Enable debug mode")
+
+}

cmd/rwp/cmd/serve/api.go

@@ -0,0 +1,257 @@
+package serve
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/gorilla/mux"
+	httprange "github.com/gotd/contrib/http_range"
+	"github.com/pkg/errors"
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache"
+	"github.com/readium/go-toolkit/pkg/asset"
+	"github.com/readium/go-toolkit/pkg/manifest"
+	"github.com/readium/go-toolkit/pkg/pub"
+	"github.com/readium/go-toolkit/pkg/streamer"
+	"github.com/zeebo/xxh3"
+)
+
+type demoListItem struct {
+	Filename string `json:"filename"`
+	Path     string `json:"path"`
+}
+
+func (s *Server) demoList(w http.ResponseWriter, req *http.Request) {
+	fi, err := os.ReadDir(s.config.BaseDirectory)
+	if err != nil {
+		slog.Error("failed reading publications directory", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+	files := make([]demoListItem, len(fi))
+	for i, f := range fi {
+		files[i] = demoListItem{
+			Filename: f.Name(),
+			Path:     base64.RawURLEncoding.EncodeToString([]byte(f.Name())),
+		}
+	}
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", s.config.JSONIndent)
+	enc.Encode(files)
+}
+
+func (s *Server) getPublication(filename string) (*pub.Publication, error) {
+	fpath, err := base64.RawURLEncoding.DecodeString(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	cp := filepath.Clean(string(fpath))
+	dat, ok := s.lfu.Get(cp)
+	if !ok {
+		pub, err := streamer.New(streamer.Config{
+			InferA11yMetadata: s.config.InferA11yMetadata,
+		}).Open(asset.File(filepath.Join(s.config.BaseDirectory, cp)), "")
+		if err != nil {
+			return nil, errors.Wrap(err, "failed opening "+cp)
+		}
+
+		// TODO: Remove this after we make links relative in the go-toolkit
+		for i, link := range pub.Manifest.Links {
+			pub.Manifest.Links[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.Resources {
+			pub.Manifest.Resources[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.ReadingOrder {
+			pub.Manifest.ReadingOrder[i] = makeRelative(link)
+		}
+		for i, link := range pub.Manifest.TableOfContents {
+			pub.Manifest.TableOfContents[i] = makeRelative(link)
+		}
+		var makeCollectionRelative func(mp manifest.PublicationCollectionMap)
+		makeCollectionRelative = func(mp manifest.PublicationCollectionMap) {
+			for i := range mp {
+				for j := range mp[i] {
+					for k := range mp[i][j].Links {
+						mp[i][j].Links[k] = makeRelative(mp[i][j].Links[k])
+					}
+					makeCollectionRelative(mp[i][j].Subcollections)
+				}
+			}
+		}
+		makeCollectionRelative(pub.Manifest.Subcollections)
+
+		// Cache the publication
+		encPub := &cache.CachedPublication{Publication: pub}
+		s.lfu.Set(cp, encPub)
+
+		return encPub.Publication, nil
+	}
+	return dat.(*cache.CachedPublication).Publication, nil
+}
+
+func (s *Server) getManifest(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	filename := vars["path"]
+
+	// Load the publication
+	publication, err := s.getPublication(filename)
+	if err != nil {
+		slog.Error("failed opening publication", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Create "self" link in manifest
+	scheme := "http://"
+	if req.TLS != nil || req.Header.Get("X-Forwarded-Proto") == "https" {
+		// Note: this is never going to be 100% accurate behind proxies,
+		// but it's better than nothing for a dev server.
+		scheme = "https://"
+	}
+	rPath, _ := s.router.Get("manifest").URLPath("path", vars["path"])
+	selfLink := &manifest.Link{
+		Rels: manifest.Strings{"self"},
+		Type: conformsToAsMimetype(publication.Manifest.Metadata.ConformsTo),
+		Href: scheme + req.Host + rPath.String(),
+	}
+
+	// Marshal the manifest
+	j, err := json.Marshal(publication.Manifest.ToMap(selfLink))
+	if err != nil {
+		slog.Error("failed marshalling manifest JSON", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Indent JSON
+	var identJSON bytes.Buffer
+	if s.config.JSONIndent == "" {
+		_, err = identJSON.Write(j)
+		if err != nil {
+			slog.Error("failed writing manifest JSON to buffer", "error", err)
+			w.WriteHeader(500)
+			return
+		}
+	} else {
+		err = json.Indent(&identJSON, j, "", s.config.JSONIndent)
+		if err != nil {
+			slog.Error("failed indenting manifest JSON", "error", err)
+			w.WriteHeader(500)
+			return
+		}
+	}
+
+	// Add headers
+	w.Header().Set("content-type", conformsToAsMimetype(publication.Manifest.Metadata.ConformsTo)+"; charset=utf-8")
+	w.Header().Set("cache-control", "private, must-revalidate")
+	w.Header().Set("access-control-allow-origin", "*") // TODO: provide options?
+
+	// Etag based on hash of the manifest bytes
+	etag := `"` + strconv.FormatUint(xxh3.Hash(identJSON.Bytes()), 36) + `"`
+	w.Header().Set("Etag", etag)
+	if match := req.Header.Get("If-None-Match"); match != "" {
+		if strings.Contains(match, etag) {
+			w.WriteHeader(http.StatusNotModified)
+			return
+		}
+	}
+
+	// Write response body
+	_, err = identJSON.WriteTo(w)
+	if err != nil {
+		slog.Error("failed writing manifest JSON to response writer", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+}
+
+func (s *Server) getAsset(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	filename := vars["path"]
+
+	// Load the publication
+	publication, err := s.getPublication(filename)
+	if err != nil {
+		slog.Error("failed opening publication", "error", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	// Make sure the asset exists in the publication
+	href := path.Clean(vars["asset"])
+	link := publication.Find(href)
+	if link == nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	// Get the asset from the publication
+	res := publication.Get(*link)
+	defer res.Close()
+
+	// Get asset length in bytes
+	l, rerr := res.Length()
+	if rerr != nil {
+		w.WriteHeader(rerr.HTTPStatus())
+		w.Write([]byte(rerr.Error()))
+		return
+	}
+
+	// Patch mimetype where necessary
+	contentType := link.MediaType().String()
+	if sub, ok := mimeSubstitutions[contentType]; ok {
+		contentType = sub
+	}
+	w.Header().Set("content-type", contentType)
+	w.Header().Set("cache-control", "private, max-age=86400, immutable")
+	w.Header().Set("content-length", strconv.FormatInt(l, 10))
+	w.Header().Set("access-control-allow-origin", "*") // TODO: provide options?
+
+	var start, end int64
+	// Range reading assets
+	rangeHeader := r.Header.Get("range")
+	if rangeHeader != "" {
+		rng, err := httprange.ParseRange(rangeHeader, l)
+		if err != nil {
+			slog.Error("failed parsing range header", "error", err)
+			w.WriteHeader(http.StatusLengthRequired)
+			return
+		}
+		if len(rng) > 1 {
+			slog.Error("no support for multiple read ranges")
+			w.WriteHeader(http.StatusNotImplemented)
+			return
+		}
+		if len(rng) > 0 {
+			w.Header().Set("content-range", rng[0].ContentRange(l))
+			start = rng[0].Start
+			end = start + rng[0].Length - 1
+			w.Header().Set("content-length", strconv.FormatInt(rng[0].Length, 10))
+		}
+	}
+	if w.Header().Get("content-range") != "" {
+		w.WriteHeader(http.StatusPartialContent)
+	}
+
+	// Stream the asset
+	_, rerr = res.Stream(w, start, end)
+	if rerr != nil {
+		if errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET) {
+			// Ignore client errors
+			return
+		}
+
+		slog.Error("failed streaming asset", "error", rerr.Error())
+	}
+
+}

cmd/rwp/cmd/serve/cache/local.go

@@ -0,0 +1,90 @@
+package cache
+
+// Originally from https://github.com/go-redis/cache/blob/v8.4.3/local.go
+// Modified to store interface{} instead of []byte
+
+import (
+	"sync"
+	"time"
+
+	"github.com/vmihailenco/go-tinylfu"
+	"golang.org/x/exp/rand"
+)
+
+type Evictable interface {
+	OnEvict()
+}
+
+type LocalCache interface {
+	Set(key string, data Evictable)
+	Get(key string) (Evictable, bool)
+	Del(key string)
+}
+
+type TinyLFU struct {
+	mu     sync.Mutex
+	rand   *rand.Rand
+	lfu    *tinylfu.T
+	ttl    time.Duration
+	offset time.Duration
+}
+
+var _ LocalCache = (*TinyLFU)(nil)
+
+func NewTinyLFU(size int, ttl time.Duration) *TinyLFU {
+	const maxOffset = 10 * time.Second
+
+	offset := ttl / 10
+	if offset > maxOffset {
+		offset = maxOffset
+	}
+
+	return &TinyLFU{
+		rand:   rand.New(rand.NewSource(uint64(time.Now().UnixNano()))),
+		lfu:    tinylfu.New(size, 100000),
+		ttl:    ttl,
+		offset: offset,
+	}
+}
+
+func (c *TinyLFU) UseRandomizedTTL(offset time.Duration) {
+	c.offset = offset
+}
+
+func (c *TinyLFU) Set(key string, b Evictable) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	ttl := c.ttl
+	if c.offset > 0 {
+		ttl += time.Duration(c.rand.Int63n(int64(c.offset)))
+	}
+
+	c.lfu.Set(&tinylfu.Item{
+		Key:      key,
+		Value:    b,
+		ExpireAt: time.Now().Add(ttl),
+		OnEvict: func() {
+			b.OnEvict()
+		},
+	})
+}
+
+func (c *TinyLFU) Get(key string) (Evictable, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	val, ok := c.lfu.Get(key)
+	if !ok {
+		return nil, false
+	}
+
+	return val.(Evictable), true
+}
+
+func (c *TinyLFU) Del(key string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.lfu.Del(key)
+}

cmd/rwp/cmd/serve/cache/pubcache.go

@@ -0,0 +1,22 @@
+package cache
+
+import (
+	"github.com/readium/go-toolkit/pkg/pub"
+)
+
+// CachedPublication implements Evictable
+type CachedPublication struct {
+	*pub.Publication
+}
+
+func EncapsulatePublication(pub *pub.Publication) *CachedPublication {
+	cp := &CachedPublication{pub}
+	return cp
+}
+
+func (cp *CachedPublication) OnEvict() {
+	// Cleanup
+	if cp.Publication != nil {
+		cp.Publication.Close()
+	}
+}

cmd/rwp/cmd/serve/helpers.go

@@ -0,0 +1,64 @@
+package serve
+
+import (
+	"strings"
+
+	"github.com/readium/go-toolkit/pkg/manifest"
+)
+
+var mimeSubstitutions = map[string]string{
+	"application/vnd.ms-opentype":          "font/otf",                                            // Not just because it's sane, but because CF will compress it!
+	"application/vnd.readium.content+json": "application/vnd.readium.content+json; charset=utf-8", // Need utf-8 encoding
+}
+
+var compressableMimes = []string{
+	"application/javascript",
+	"application/x-javascript",
+	"image/x-icon",
+	"text/css",
+	"text/html",
+	"application/xhtml+xml",
+	"application/webpub+json",
+	"application/divina+json",
+	"application/vnd.readium.position-list+json",
+	"application/vnd.readium.content+json",
+	"application/audiobook+json",
+	"font/ttf",
+	"application/ttf",
+	"application/x-ttf",
+	"application/x-font-ttf",
+	"font/otf",
+	"application/otf",
+	"application/x-otf",
+	"application/vnd.ms-opentype",
+	"font/opentype",
+	"application/opentype",
+	"application/x-opentype",
+	"application/truetype",
+	"application/font-woff",
+	"font/x-woff",
+	"application/vnd.ms-fontobject",
+}
+
+func makeRelative(link manifest.Link) manifest.Link {
+	link.Href = strings.TrimPrefix(link.Href, "/")
+	for i, alt := range link.Alternates {
+		link.Alternates[i].Href = strings.TrimPrefix(alt.Href, "/")
+	}
+	return link
+}
+
+func conformsToAsMimetype(conformsTo manifest.Profiles) string {
+	mime := "application/webpub+json"
+	for _, profile := range conformsTo {
+		if profile == manifest.ProfileDivina {
+			mime = "application/divina+json"
+		} else if profile == manifest.ProfileAudiobook {
+			mime = "application/audiobook+json"
+		} else {
+			continue
+		}
+		break
+	}
+	return mime
+}

cmd/rwp/cmd/serve/router.go

@@ -0,0 +1,47 @@
+package serve
+
+import (
+	"net/http"
+	"net/http/pprof"
+
+	"github.com/CAFxX/httpcompression"
+	"github.com/gorilla/mux"
+)
+
+func (s *Server) Routes() *mux.Router {
+	r := mux.NewRouter()
+
+	r.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+
+	if s.config.Debug {
+		r.HandleFunc("/debug/pprof/", pprof.Index)
+		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+		r.Handle("/debug/pprof/allocs", pprof.Handler("allocs"))
+		r.Handle("/debug/pprof/block", pprof.Handler("block"))
+		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		r.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
+		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+	}
+
+	r.HandleFunc("/list.json", s.demoList).Name("demo_list")
+
+	pub := r.PathPrefix("/{path}").Subrouter()
+	// TODO: publication loading middleware with pub.Use()
+	pub.Use(func(h http.Handler) http.Handler {
+		adapter, _ := httpcompression.DefaultAdapter(httpcompression.ContentTypes(compressableMimes, false))
+		return adapter(h)
+	})
+	pub.HandleFunc("/manifest.json", s.getManifest).Name("manifest")
+	pub.HandleFunc("/{asset:.*}", s.getAsset).Name("asset")
+
+	s.router = r
+	return r
+}

cmd/rwp/cmd/serve/server.go

@@ -0,0 +1,32 @@
+package serve
+
+import (
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/readium/go-toolkit/cmd/rwp/cmd/serve/cache"
+	"github.com/readium/go-toolkit/pkg/streamer"
+)
+
+type ServerConfig struct {
+	Debug             bool
+	BaseDirectory     string
+	JSONIndent        string
+	InferA11yMetadata streamer.InferA11yMetadata
+}
+
+type Server struct {
+	config ServerConfig
+	router *mux.Router
+	lfu    *cache.TinyLFU
+}
+
+const MaxCachedPublicationAmount = 10
+const MaxCachedPublicationTTL = time.Second * time.Duration(600)
+
+func NewServer(config ServerConfig) *Server {
+	return &Server{
+		config: config,
+		lfu:    cache.NewTinyLFU(MaxCachedPublicationAmount, MaxCachedPublicationTTL),
+	}
+}

go.mod

@@ -3,10 +3,12 @@ module github.com/readium/go-toolkit
 go 1.21
 
 require (
+	github.com/CAFxX/httpcompression v0.0.9
 	github.com/agext/regexp v1.3.0
 	github.com/andybalholm/cascadia v1.3.2
 	github.com/deckarep/golang-set v1.7.1
 	github.com/gorilla/mux v1.7.4
+	github.com/gotd/contrib v0.20.0
 	github.com/opds-community/libopds2-go v0.0.0-20170628075933-9c163cf60f6e
 	github.com/pdfcpu/pdfcpu v0.5.0
 	github.com/pkg/errors v0.9.1
@@ -16,34 +18,40 @@ require (
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.9.0
 	github.com/trimmer-io/go-xmp v1.0.0
 	github.com/urfave/negroni v1.0.0
-	golang.org/x/net v0.10.0
-	golang.org/x/text v0.12.0
+	github.com/vmihailenco/go-tinylfu v0.2.2
+	github.com/zeebo/xxh3 v1.0.2
+	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10
+	golang.org/x/net v0.23.0
+	golang.org/x/text v0.14.0
 )
 
 require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antchfx/xpath v1.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hhrutter/lzw v1.0.0 // indirect
 	github.com/hhrutter/tiff v1.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	golang.org/x/image v0.11.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )