[WIP] Fly deployment

.gitignore

@@ -39,3 +39,5 @@ next-env.d.ts
 dbs/
 tls/
 dist/
+
+.env

apps/certbot-service/.env.example

@@ -0,0 +1,9 @@
+CERTBOT_DOMAIN=db.postgres.new
+CERTBOT_EMAIL=
+CLOUDFLARE_API_TOKEN=
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_ENDPOINT_URL_S3=http://minio:9000
+AWS_REGION=us-east-1
+AWS_SECRET_ACCESS_KEY=minioadmin
+BUCKET_NAME=test
+S3FS_MOUNT=/mnt/s3

apps/certbot-service/.gitignore

@@ -0,0 +1 @@
+.env

apps/certbot-service/Dockerfile

@@ -0,0 +1,47 @@
+# syntax = docker/dockerfile:1
+
+# Adjust CERTBOT_VERSION as desired
+ARG CERTBOT_VERSION=2.11.0
+FROM certbot/dns-cloudflare:v${CERTBOT_VERSION} as base
+
+WORKDIR /app
+
+# Build S3FS
+FROM base as build-s3fs
+
+# Install dependencies
+RUN apk add --no-cache \
+  git \
+  build-base \
+  automake \
+  autoconf \
+  libxml2-dev \
+  fuse-dev \
+  curl-dev
+
+RUN git clone https://github.com/s3fs-fuse/s3fs-fuse.git --branch v1.94 && \
+  cd s3fs-fuse && \
+  ./autogen.sh && \
+  ./configure && \
+  make && \
+  make install
+
+# Final stage
+FROM base
+
+# Install dependencies
+RUN apk add --no-cache \
+  bash \
+  curl \
+  fuse \
+  libxml2
+
+COPY --from=build-s3fs /usr/local/bin/s3fs /usr/local/bin/s3fs
+COPY certbot.sh deploy-hook.sh entrypoint.sh /app/
+
+RUN chmod +x certbot.sh
+RUN chmod +x deploy-hook.sh
+
+ENTRYPOINT [ "./entrypoint.sh" ]
+
+CMD [ "./certbot.sh" ]

apps/certbot-service/README.md

@@ -0,0 +1,105 @@
+# Certbot
+
+This service is responsible for managing the certificates for the PGLite instances.
+
+It uses `fly machine run --schedule weekly` to wake up the service every week to renew the certificates if needed. Let's Encrypt certificates are valid for 90 days.
+
+## Testing certbot-service locally
+
+Copy `.env.example` to `.env` and set the missing environment variables.
+
+Start minio to emulate the S3 service:
+
+```shell
+docker compose up -d minio    
+```
+
+Initialize the bucket:
+
+```shell
+docker compose up minio-init
+```
+
+Build and run the certbot service:
+
+```shell
+docker compose up --build certbot-service
+```
+
+The certificates will be generated in `/mnt/s3/tls`.
+
+## Deploying to fly.io
+
+1. Create a new app if it doesn't exist
+
+```shell
+flyctl apps create postgres-new-certbot
+```
+
+2. Build and deploy the Docker image to fly.io image registry
+
+```shell
+flyctl deploy --build-only --push -a postgres-new-certbot --image-label
+ latest
+```
+
+3. Set the appropriate environment variables and secrets for the app "postgres-new-certbot" (see `.env.example`) in fly.io UI (available in Bitwarden as a secure note "fly.io postgres.new cerbot .env")
+
+4. Setup [cron-manager](https://github.com/fly-apps/cron-manager?tab=readme-ov-file#getting-started) to run the certbot service every 2 weeks with the following `schedules.json`:
+
+```json
+[
+  {
+      "name": "postgres-new-certbot",
+      "app_name": "postgres-new-certbot",
+      "schedule": "0 0 1,15 * *",
+      "region": "ord",
+      "command": "./certbot.sh",
+      "command_timeout": 120,
+      "enabled": true,
+      "config": {
+          "metadata": {
+              "fly_process_group": "cron"
+          },
+          "auto_destroy": true,
+          "disable_machine_autostart": true,
+          "guest": {
+              "cpu_kind": "shared",
+              "cpus": 1,
+              "memory_mb": 256
+          },
+          "image": "registry.fly.io/postgres-new-certbot:latest",
+          "restart": {
+              "max_retries": 1,
+              "policy": "no"
+          }
+      }
+  }
+]
+```
+
+5. Test running the job by SSHing into cron-manager console
+
+Run this command in the cron-manager root folder:
+
+```shell
+flyctl ssh console
+```
+
+Once in the cron-manager instance:
+
+```shell
+cm jobs trigger 1
+```
+
+If you open the "postgres-new-certbot" live logs in fly.io UI, you should see the job being executed.
+
+6. You can check if the certificates are present in the Tigris bucket
+
+Run this command in the apps/db-instance folder:
+
+```shell
+flyctl storage dashboard
+```
+
+It should open the Tigris dashboard where you can check the bucket's content. The certificates should be created under `/tls`.

apps/certbot-service/certbot.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+CONFIG_DIR="${S3FS_MOUNT}/tls/letsencrypt"
+CERT_PATH="${CONFIG_DIR}/live/${CERTBOT_DOMAIN}/fullchain.pem"
+CLOUD_FLARE_INI="/app/cloudflare.ini"
+DEPLOY_HOOK="/app/deploy-hook.sh"
+
+renew_certificate() {
+    echo "Certificates exist. Renewing..."
+    certbot renew --non-interactive \
+        --dns-cloudflare \
+        --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
+        --deploy-hook "${DEPLOY_HOOK}" \
+        --config-dir "${CONFIG_DIR}"
+}
+
+create_certificate() {
+    echo "Certificates do not exist. Creating..."
+    certbot certonly --non-interactive \
+        --agree-tos \
+        --email "${CERTBOT_EMAIL}" \
+        --dns-cloudflare \
+        --dns-cloudflare-credentials "${CLOUD_FLARE_INI}" \
+        --dns-cloudflare-propagation-seconds 60 \
+        -d "*.${CERTBOT_DOMAIN}" \
+        --deploy-hook "${DEPLOY_HOOK}" \
+        --config-dir "${CONFIG_DIR}"
+}
+
+main() {
+    if [[ -f "${CERT_PATH}" ]]; then
+        renew_certificate
+    else
+        create_certificate
+    fi
+}
+
+main "$@"

apps/certbot-service/deploy-hook.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SOURCE_DIR="$S3FS_MOUNT/tls/letsencrypt/live/$CERTBOT_DOMAIN"
+TARGET_DIR="$S3FS_MOUNT/tls"
+
+# Ensure the target directory exists
+mkdir -p $TARGET_DIR
+
+# Copy the key and cert to the target directory
+cp -f $SOURCE_DIR/privkey.pem $TARGET_DIR/key.pem
+cp -f $SOURCE_DIR/fullchain.pem $TARGET_DIR/cert.pem

apps/certbot-service/docker-compose.yml

@@ -0,0 +1,47 @@
+services:
+  certbot-service:
+    image: certbot-service
+    build:
+      context: .
+    environment:
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_ENDPOINT_URL_S3: ${AWS_ENDPOINT_URL_S3}
+      AWS_REGION: ${AWS_REGION}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      BUCKET_NAME: ${BUCKET_NAME}
+      CERTBOT_DOMAIN: ${CERTBOT_DOMAIN}
+      CERTBOT_EMAIL: ${CERTBOT_EMAIL}
+      CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN}
+      S3FS_MOUNT: ${S3FS_MOUNT}
+    ports:
+      - 5432:5432
+    devices:
+      - /dev/fuse
+    cap_add:
+      - SYS_ADMIN
+    depends_on:
+      minio:
+        condition: service_healthy
+  minio:
+    image: minio/minio
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    ports:
+      - 9000:9000
+    command: server /data
+    healthcheck:
+      test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 1
+  minio-init:
+    image: minio/mc
+    entrypoint: >
+      /bin/sh -c "
+        mc alias set local http://minio:9000 minioadmin minioadmin;
+        (mc ls local/test || mc mb local/test);
+      "
+    depends_on:
+      minio:
+        condition: service_healthy

apps/certbot-service/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# s3fs ################################
+cleanup() {
+  echo "Unmounting s3fs..."
+  fusermount -u $S3FS_MOUNT
+  exit 0
+}
+
+forward_signal() {
+  kill -$1 "$MAIN_PID"
+}
+
+trap 'forward_signal SIGINT' SIGINT
+trap 'forward_signal SIGTERM' SIGTERM
+trap 'cleanup' EXIT
+
+# Create the mount point directory
+mkdir -p $S3FS_MOUNT
+
+# Mount the S3 bucket
+s3fs $BUCKET_NAME $S3FS_MOUNT -o use_path_request_style -o url=$AWS_ENDPOINT_URL_S3 -o endpoint=$AWS_REGION
+
+# Check if the mount was successful
+if mountpoint -q $S3FS_MOUNT; then
+  echo "S3 bucket mounted successfully at $S3FS_MOUNT"
+else
+  echo "Failed to mount S3 bucket"
+  exit 1
+fi
+
+# cloudflare.ini ######################
+echo "dns_cloudflare_api_token = $CLOUDFLARE_API_TOKEN" > /app/cloudflare.ini
+chmod 600 /app/cloudflare.ini
+
+# Execute the original command
+"$@" &
+MAIN_PID=$!
+
+wait $MAIN_PID

apps/certbot-service/fly.toml

@@ -0,0 +1,4 @@
+app = 'postgres-new-certbot'
+primary_region = 'yyz'
+
+[build]

apps/db-service/.env.example

@@ -0,0 +1,6 @@
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_ENDPOINT_URL_S3=http://minio:9000
+AWS_REGION=us-east-1
+AWS_SECRET_ACCESS_KEY=minioadmin
+BUCKET_NAME=test
+S3FS_MOUNT=/mnt/s3

apps/db-service/README.md

@@ -99,3 +99,11 @@ To stop all Docker containers, run:
 ```shell
 docker compose down
 ```
+
+# Fly deployment
+
+```shell
+flyctl launch --org supabase-dev
+```
+
+In the browser, select "enable Tigris" and confirm.

apps/db-service/docker-compose.yml

@@ -3,13 +3,8 @@ services:
     image: db-service
     build:
       context: .
-    environment:
-      S3FS_ENDPOINT: http://minio:9000
-      S3FS_BUCKET: test
-      S3FS_REGION: us-east-1 # default region for s3-compatible APIs
-      S3FS_MOUNT: /mnt/s3
-      AWS_ACCESS_KEY_ID: minioadmin
-      AWS_SECRET_ACCESS_KEY: minioadmin
+    env_file:
+      - .env
     ports:
       - 5432:5432
     devices:
@@ -23,13 +18,8 @@ services:
     image: tls-init
     build:
       context: .
-    environment:
-      S3FS_ENDPOINT: http://minio:9000
-      S3FS_BUCKET: test
-      S3FS_REGION: us-east-1 # default region for s3-compatible APIs
-      S3FS_MOUNT: /mnt/s3
-      AWS_ACCESS_KEY_ID: minioadmin
-      AWS_SECRET_ACCESS_KEY: minioadmin
+    env_file:
+      - .env
     devices:
       - /dev/fuse
     cap_add:

apps/db-service/entrypoint.sh

@@ -21,7 +21,7 @@ trap 'cleanup' EXIT
 mkdir -p $S3FS_MOUNT
 
 # Mount the S3 bucket
-s3fs $S3FS_BUCKET $S3FS_MOUNT -o use_path_request_style -o url=$S3FS_ENDPOINT -o endpoint=$S3FS_REGION
+s3fs $BUCKET_NAME $S3FS_MOUNT -o url=$AWS_ENDPOINT_URL_S3 -o use_path_request_style
 
 # Check if the mount was successful
 if mountpoint -q $S3FS_MOUNT; then