feat: blob, web3.storage and ucan conclude capabilities together with api handlers (#1342)

Adds implementation of `blob/*`, `web3.storage/*` and `ucan/conclude`
handlers and capabilities.

Author: vasco-santos

packages/capabilities/package.json

@@ -56,6 +56,10 @@
       "types": "./dist/src/filecoin/dealer.d.ts",
       "import": "./src/filecoin/dealer.js"
     },
+    "./web3.storage/blob": {
+      "types": "./dist/src/web3.storage/blob.d.ts",
+      "import": "./src/web3.storage/blob.js"
+    },
     "./types": {
       "types": "./dist/src/types.d.ts",
       "import": "./src/types.js"
@@ -88,7 +92,8 @@
     "@ucanto/principal": "^9.0.1",
     "@ucanto/transport": "^9.1.1",
     "@ucanto/validator": "^9.0.2",
-    "@web3-storage/data-segment": "^3.2.0"
+    "@web3-storage/data-segment": "^3.2.0",
+    "uint8arrays": "^5.0.3"
   },
   "devDependencies": {
     "@web3-storage/eslint-config-w3up": "workspace:^",

packages/capabilities/src/blob.js

@@ -0,0 +1,75 @@
+/**
+ * Blob Capabilities.
+ *
+ * Blob is a fixed size byte array addressed by the multihash.
+ * Usually blobs are used to represent set of IPLD blocks at different byte ranges.
+ *
+ * These can be imported directly with:
+ * ```js
+ * import * as Blob from '@web3-storage/capabilities/blob'
+ * ```
+ *
+ * @module
+ */
+import { capability, Schema } from '@ucanto/validator'
+import { equalBlob, equalWith, SpaceDID } from './utils.js'
+
+/**
+ * Agent capabilities for Blob protocol
+ */
+
+/**
+ * Capability can only be delegated (but not invoked) allowing audience to
+ * derived any `blob/` prefixed capability for the (memory) space identified
+ * by DID in the `with` field.
+ */
+export const blob = capability({
+  can: 'blob/*',
+  /**
+   * DID of the (memory) space where Blob is intended to
+   * be stored.
+   */
+  with: SpaceDID,
+  derives: equalWith,
+})
+
+/**
+ * Blob description for being ingested by the service.
+ */
+export const content = Schema.struct({
+  /**
+   * A multihash digest of the blob payload bytes, uniquely identifying blob.
+   */
+  digest: Schema.bytes(),
+  /**
+   * Number of bytes contained by this blob. Service will provision write target
+   * for this exact size. Attempt to write a larger Blob file will fail.
+   */
+  size: Schema.integer(),
+})
+
+/**
+ * `blob/add` capability allows agent to store a Blob into a (memory) space
+ * identified by did:key in the `with` field. Agent should compute blob multihash
+ * and size and provide it under `nb.blob` field, allowing a service to provision
+ * a write location for the agent to PUT desired Blob into.
+ */
+export const add = capability({
+  can: 'blob/add',
+  /**
+   * DID of the (memory) space where Blob is intended to
+   * be stored.
+   */
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * Blob to be added on the space.
+     */
+    blob: content,
+  }),
+  derives: equalBlob,
+})
+
+// ⚠️ We export imports here so they are not omitted in generated typedefs
+// @see https://github.com/microsoft/TypeScript/issues/51548
+export { Schema }

packages/capabilities/src/http.js

@@ -0,0 +1,49 @@
+/**
+ * HTTP Capabilities.
+ *
+ * These can be imported directly with:
+ * ```js
+ * import * as HTTP from '@web3-storage/capabilities/http'
+ * ```
+ *
+ * @module
+ */
+import { capability, Schema, ok } from '@ucanto/validator'
+import { content } from './blob.js'
+import { equal, equalBody, equalWith, SpaceDID, Await, and } from './utils.js'
+
+/**
+ * `http/put` capability invocation MAY be performed by any authorized agent on behalf of the subject
+ * as long as they have referenced `body` content to do so.
+ */
+export const put = capability({
+  can: 'http/put',
+  /**
+   * DID of the (memory) space where Blob is intended to
+   * be stored.
+   */
+  with: SpaceDID,
+  nb: Schema.struct({
+    /**
+     * Description of body to send (digest/size).
+     */
+    body: content,
+    /**
+     * HTTP(S) location that can receive blob content via HTTP PUT request.
+     */
+    url: Schema.string().or(Await),
+    /**
+     * HTTP headers.
+     */
+    headers: Schema.dictionary({ value: Schema.string() }).or(Await),
+  }),
+  derives: (claim, from) => {
+    return (
+      and(equalWith(claim, from)) ||
+      and(equalBody(claim, from)) ||
+      and(equal(claim.nb.url, from.nb, 'url')) ||
+      and(equal(claim.nb.headers, from.nb, 'headers')) ||
+      ok({})
+    )
+  },
+})

packages/capabilities/src/index.js

@@ -19,6 +19,9 @@ import * as DealTracker from './filecoin/deal-tracker.js'
 import * as UCAN from './ucan.js'
 import * as Plan from './plan.js'
 import * as Usage from './usage.js'
+import * as Blob from './blob.js'
+import * as W3sBlob from './web3.storage/blob.js'
+import * as HTTP from './http.js'
 
 export {
   Access,
@@ -63,6 +66,7 @@ export const abilitiesAsStrings = [
   Access.access.can,
   Access.authorize.can,
   UCAN.attest.can,
+  UCAN.conclude.can,
   Customer.get.can,
   Consumer.has.can,
   Consumer.get.can,
@@ -86,4 +90,10 @@ export const abilitiesAsStrings = [
   Plan.get.can,
   Usage.usage.can,
   Usage.report.can,
+  Blob.blob.can,
+  Blob.add.can,
+  W3sBlob.blob.can,
+  W3sBlob.allocate.can,
+  W3sBlob.accept.can,
+  HTTP.put.can,
 ]

packages/capabilities/src/types.ts

@@ -21,6 +21,9 @@ import {
 import { space, info } from './space.js'
 import * as provider from './provider.js'
 import { top } from './top.js'
+import * as BlobCaps from './blob.js'
+import * as W3sBlobCaps from './web3.storage/blob.js'
+import * as HTTPCaps from './http.js'
 import * as StoreCaps from './store.js'
 import * as UploadCaps from './upload.js'
 import * as AccessCaps from './access.js'
@@ -41,6 +44,10 @@ export type ISO8601Date = string
 
 export type { Unit, PieceLink }
 
+export interface UCANAwait<Selector extends string = string, Task = unknown> {
+  'ucan/await': [Selector, Link<Task>]
+}
+
 /**
  * An IPLD Link that has the CAR codec code.
  */
@@ -439,6 +446,95 @@ export interface UploadNotFound extends Ucanto.Failure {
 
 export type UploadGetFailure = UploadNotFound | Ucanto.Failure
 
+// HTTP
+export type HTTPPut = InferInvokedCapability<typeof HTTPCaps.put>
+
+// Blob
+export type Blob = InferInvokedCapability<typeof BlobCaps.blob>
+export type BlobAdd = InferInvokedCapability<typeof BlobCaps.add>
+export type ServiceBlob = InferInvokedCapability<typeof W3sBlobCaps.blob>
+export type BlobAllocate = InferInvokedCapability<typeof W3sBlobCaps.allocate>
+export type BlobAccept = InferInvokedCapability<typeof W3sBlobCaps.accept>
+
+export type BlobMultihash = Uint8Array
+export interface BlobModel {
+  digest: BlobMultihash
+  size: number
+}
+
+// Blob add
+export interface BlobAddSuccess {
+  site: UCANAwait<'.out.ok.site'>
+}
+
+export interface BlobSizeOutsideOfSupportedRange extends Ucanto.Failure {
+  name: 'BlobSizeOutsideOfSupportedRange'
+}
+
+export interface AwaitError extends Ucanto.Failure {
+  name: 'AwaitError'
+}
+
+// TODO: We need Ucanto.Failure because provideAdvanced can't handle errors without it
+export type BlobAddFailure =
+  | BlobSizeOutsideOfSupportedRange
+  | AwaitError
+  | StorageGetError
+  | Ucanto.Failure
+
+export interface BlobListItem {
+  blob: BlobModel
+  insertedAt: ISO8601Date
+}
+
+// Blob allocate
+export interface BlobAllocateSuccess {
+  size: number
+  address?: BlobAddress
+}
+
+export interface BlobAddress {
+  url: ToString<URL>
+  headers: Record<string, string>
+  expiresAt: ISO8601Date
+}
+
+// If user space has not enough space to allocate the blob.
+export interface NotEnoughStorageCapacity extends Ucanto.Failure {
+  name: 'NotEnoughStorageCapacity'
+}
+
+export type BlobAllocateFailure = NotEnoughStorageCapacity | Ucanto.Failure
+
+// Blob accept
+export interface BlobAcceptSuccess {
+  // A Link for a delegation with site commiment for the added blob.
+  site: Link
+}
+
+export interface AllocatedMemoryHadNotBeenWrittenTo extends Ucanto.Failure {
+  name: 'AllocatedMemoryHadNotBeenWrittenTo'
+}
+
+// TODO: We should type the store errors and add them here, instead of Ucanto.Failure
+export type BlobAcceptFailure =
+  | AllocatedMemoryHadNotBeenWrittenTo
+  | Ucanto.Failure
+
+// Storage errors
+export type StoragePutError = StorageOperationError
+export type StorageGetError = StorageOperationError | RecordNotFound
+
+// Operation on a storage failed with unexpected error
+export interface StorageOperationError extends Error {
+  name: 'StorageOperationFailed'
+}
+
+// Record requested not found in the storage
+export interface RecordNotFound extends Error {
+  name: 'RecordNotFound'
+}
+
 // Store
 export type Store = InferInvokedCapability<typeof StoreCaps.store>
 export type StoreAdd = InferInvokedCapability<typeof StoreCaps.add>
@@ -530,6 +626,7 @@ export interface UploadListSuccess extends ListResponse<UploadListItem> {}
 
 export type UCANRevoke = InferInvokedCapability<typeof UCANCaps.revoke>
 export type UCANAttest = InferInvokedCapability<typeof UCANCaps.attest>
+export type UCANConclude = InferInvokedCapability<typeof UCANCaps.conclude>
 
 export interface Timestamp {
   /**
@@ -540,6 +637,8 @@ export interface Timestamp {
 
 export type UCANRevokeSuccess = Timestamp
 
+export type UCANConcludeSuccess = Timestamp
+
 /**
  * Error is raised when `UCAN` being revoked is not supplied or it's proof chain
  * leading to supplied `scope` is not supplied.
@@ -578,6 +677,15 @@ export type UCANRevokeFailure =
   | UnauthorizedRevocation
   | RevocationsStoreFailure
 
+/**
+ * Error is raised when receipt is received for unknown invocation
+ */
+export interface ReferencedInvocationNotFound extends Ucanto.Failure {
+  name: 'ReferencedInvocationNotFound'
+}
+
+export type UCANConcludeFailure = ReferencedInvocationNotFound | Ucanto.Failure
+
 // Admin
 export type Admin = InferInvokedCapability<typeof AdminCaps.admin>
 export type AdminUploadInspect = InferInvokedCapability<
@@ -686,6 +794,7 @@ export type ServiceAbilityArray = [
   Access['can'],
   AccessAuthorize['can'],
   UCANAttest['can'],
+  UCANConclude['can'],
   CustomerGet['can'],
   ConsumerHas['can'],
   ConsumerGet['can'],
@@ -708,7 +817,13 @@ export type ServiceAbilityArray = [
   AdminStoreInspect['can'],
   PlanGet['can'],
   Usage['can'],
-  UsageReport['can']
+  UsageReport['can'],
+  Blob['can'],
+  BlobAdd['can'],
+  ServiceBlob['can'],
+  BlobAllocate['can'],
+  BlobAccept['can'],
+  HTTPPut['can']
 ]
 
 /**

packages/capabilities/src/ucan.js

@@ -2,7 +2,7 @@
  * UCAN core capabilities.
  */
 
-import { capability, Schema } from '@ucanto/validator'
+import { capability, Schema, ok } from '@ucanto/validator'
 import * as API from '@ucanto/interface'
 import { equalWith, equal, and, checkLink } from './utils.js'
 
@@ -74,6 +74,34 @@ export const revoke = capability({
     ),
 })
 
+/**
+ * `ucan/conclude` capability represents a receipt using a special UCAN capability.
+ *
+ * The UCAN invocation specification defines receipt record, that is cryptographically
+ * signed description of the invocation output and requested effects. Receipt
+ * structure is very similar to UCAN except it has no notion of expiry nor it is
+ * possible to delegate ability to issue receipt to another principal.
+ */
+export const conclude = capability({
+  can: 'ucan/conclude',
+  /**
+   * DID of the principal representing the Conclusion Authority.
+   * MUST be the DID of the audience of the ran invocation.
+   */
+  with: Schema.did(),
+  nb: Schema.struct({
+    /**
+     * CID of the content with the Receipt.
+     */
+    receipt: Schema.link(),
+  }),
+  derives: (claim, from) =>
+    // With field MUST be the same
+    and(equalWith(claim, from)) ||
+    and(checkLink(claim.nb.receipt, from.nb.receipt, 'nb.receipt')) ||
+    ok({}),
+})
+
 /**
  * Issued by trusted authority (usually the one handling invocation) that attest
  * that specific UCAN delegation has been considered authentic.