Skip to content

Commit c810735

Browse files
authored
feat: router (#11)
This PR adds a router interface: ```ts /** * The routing service is responsible for selecting storage nodes to allocate * blobs with. */ export interface RoutingService { /** * Selects a candidate for blob allocation from the current list of available * storage nodes. */ selectStorageProvider( digest: MultihashDigest, size: number ): Promise<Result<Principal, CandidateUnavailable | Failure>> /** * Returns information required to make an invocation to the requested storage * node. */ configureInvocation<C extends BlobAllocate | BlobAccept>( provider: Principal, capability: C, options?: Omit<UCANOptions, 'audience'> ): Promise<Result<Configuration<C>, ProofUnavailable | Failure>> } ``` Tests implement a storage provider node and setup 2 nodes that are used in tests that the routing service picks randomly from. However the router will be sticky when asked to store the same blob multiple times. This allows existing tests for allocation and accept responses to pass. Note: we need to review if the semantics for allocate and accept responses really makes sense in a context where there are multiple storage providers (TLDR; the allocation size does not).
2 parents ac49094 + ae0508c commit c810735

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+5173
-5537
lines changed

.github/workflows/w3up-client.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ jobs:
3737
node-version: ${{ matrix.node_version }}
3838
registry-url: https://registry.npmjs.org/
3939
cache: 'pnpm'
40-
- run: pnpm --filter '@storacha/client' install
40+
- run: pnpm --filter '@storacha/client...' install
4141
- run: pnpm --filter '@storacha/client' attw
4242
- uses: ./packages/w3up-client/.github/actions/test
4343
with:

packages/access-client/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,15 @@
101101
"@ipld/car": "^5.1.1",
102102
"@ipld/dag-ucan": "^3.4.0",
103103
"@scure/bip39": "^1.2.1",
104+
"@storacha/capabilities": "workspace:^",
105+
"@storacha/did-mailto": "workspace:^",
104106
"@storacha/one-webcrypto": "^1.0.1",
105107
"@ucanto/client": "^9.0.1",
106108
"@ucanto/core": "^10.0.1",
107109
"@ucanto/interface": "^10.0.1",
108110
"@ucanto/principal": "^9.0.1",
109111
"@ucanto/transport": "^9.1.1",
110112
"@ucanto/validator": "^9.0.2",
111-
"@storacha/capabilities": "workspace:^",
112-
"@storacha/did-mailto": "workspace:^",
113113
"bigint-mod-arith": "^3.1.2",
114114
"conf": "11.0.2",
115115
"multiformats": "^12.1.2",
@@ -118,6 +118,7 @@
118118
"uint8arrays": "^4.0.6"
119119
},
120120
"devDependencies": {
121+
"@storacha/eslint-config": "workspace:^",
121122
"@types/assert": "^1.5.6",
122123
"@types/inquirer": "^9.0.4",
123124
"@types/mocha": "^10.0.1",
@@ -126,7 +127,6 @@
126127
"@types/varint": "^6.0.1",
127128
"@types/ws": "^8.5.4",
128129
"@ucanto/server": "^10.0.0",
129-
"@storacha/eslint-config": "workspace:^",
130130
"assert": "^2.0.0",
131131
"mocha": "^10.2.0",
132132
"playwright-test": "^12.3.4",

packages/access-client/src/agent.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -577,7 +577,7 @@ export class Agent {
577577
// @ts-ignore
578578
capability: cap.create({
579579
with: space,
580-
nb: options.nb,
580+
nb: 'nb' in options ? options.nb : undefined,
581581
}),
582582
issuer: this.issuer,
583583
proofs: [...proofs],

packages/access-client/src/types.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ export type InvokeOptions<
249249
Match<{ can: A; with: R & Resource; nb: Caveats }, UnknownMatch>
250250
>
251251
> = UCANBasicOptions &
252-
InferNb<InferInvokedCapability<CAP>['nb']> & {
252+
Omit<InferInvokedCapability<CAP>, 'can' | 'with'> & {
253253
/**
254254
* Resource for the capability, normally a Space DID
255255
* Defaults to the current selected Space

packages/blob-index/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,17 @@
4141
},
4242
"dependencies": {
4343
"@ipld/dag-cbor": "^9.0.6",
44+
"@storacha/capabilities": "workspace:^",
4445
"@storacha/one-webcrypto": "^1.0.1",
4546
"@ucanto/core": "^10.0.1",
4647
"@ucanto/interface": "^10.0.1",
47-
"@storacha/capabilities": "workspace:^",
4848
"carstream": "^2.1.0",
4949
"multiformats": "^13.0.1",
5050
"uint8arrays": "^5.0.3"
5151
},
5252
"devDependencies": {
53-
"@ucanto/transport": "^9.1.1",
5453
"@storacha/eslint-config": "workspace:^",
54+
"@ucanto/transport": "^9.1.1",
5555
"c8": "^7.14.0",
5656
"entail": "^2.1.2",
5757
"typescript": "5.2.2"

packages/capabilities/package.json

+12-4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@
3737
"types": "./dist/test/helpers/*.d.ts",
3838
"import": "./test/helpers/*.js"
3939
},
40+
"./blob": {
41+
"types": "./dist/src/blob.d.ts",
42+
"import": "./src/blob.js"
43+
},
4044
"./filecoin": {
4145
"types": "./dist/src/filecoin/index.d.ts",
4246
"import": "./src/filecoin/index.js"
@@ -57,9 +61,13 @@
5761
"types": "./dist/src/filecoin/dealer.d.ts",
5862
"import": "./src/filecoin/dealer.js"
5963
},
60-
"./index": {
61-
"types": "./dist/src/index/index.d.ts",
62-
"import": "./src/index/index.js"
64+
"./space/index": {
65+
"types": "./dist/src/space/index.d.ts",
66+
"import": "./src/space/index.js"
67+
},
68+
"./space/blob": {
69+
"types": "./dist/src/space/blob.d.ts",
70+
"import": "./src/space/blob.js"
6371
},
6472
"./web3.storage/blob": {
6573
"types": "./dist/src/web3.storage/blob.d.ts",
@@ -101,10 +109,10 @@
101109
"uint8arrays": "^5.0.3"
102110
},
103111
"devDependencies": {
112+
"@storacha/eslint-config": "workspace:^",
104113
"@types/assert": "^1.5.6",
105114
"@types/mocha": "^10.0.0",
106115
"@types/node": "^20.8.4",
107-
"@storacha/eslint-config": "workspace:^",
108116
"assert": "^2.0.0",
109117
"mocha": "^10.2.0",
110118
"playwright-test": "^12.3.4",

packages/capabilities/readme.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import * as Filecoin from '@storacha/capabilities/filecoin'
3232
import * as Aggregator from '@storacha/capabilities/filecoin/aggregator'
3333
import * as DealTracker from '@storacha/capabilities/filecoin/deal-tracker'
3434
import * as Dealer from '@storacha/capabilities/filecoin/dealer'
35-
import * as Index from '@storacha/capabilities/index'
35+
import * as Index from '@storacha/capabilities/space/index'
3636

3737
// This package has a "main" entrypoint but we recommend the usage of the specific imports above
3838
```

packages/capabilities/src/blob.js

+56-144
Original file line numberDiff line numberDiff line change
@@ -1,177 +1,89 @@
11
/**
22
* Blob Capabilities.
33
*
4-
* Blob is a fixed size byte array addressed by the multihash.
5-
* Usually blobs are used to represent set of IPLD blocks at different byte ranges.
4+
* The blob protocol allows authorized agents allocate memory space on a storage
5+
* node and subsequently verify the content has been accepted by / delivered to
6+
* said node.
67
*
78
* These can be imported directly with:
89
* ```js
9-
* import * as Blob from '@storacha/capabilities/blob'
10+
* import * as Index from '@storacha/capabilities/blob'
1011
* ```
1112
*
1213
* @module
14+
* @see https://github.com/storacha/specs/blob/main/w3-blob.md
1315
*/
14-
import { equals } from 'uint8arrays/equals'
15-
import { capability, Schema, fail, ok } from '@ucanto/validator'
16-
import { equalBlob, equalWith, SpaceDID } from './utils.js'
17-
18-
/**
19-
* Agent capabilities for Blob protocol
20-
*/
16+
import { capability, Schema, Link, ok } from '@ucanto/validator'
17+
import { content } from './space/blob.js'
18+
import {
19+
equalBlob,
20+
equalWith,
21+
SpaceDID,
22+
and,
23+
equal,
24+
checkLink,
25+
Await,
26+
} from './utils.js'
2127

2228
/**
2329
* Capability can only be delegated (but not invoked) allowing audience to
24-
* derived any `space/blob/` prefixed capability for the (memory) space identified
25-
* by DID in the `with` field.
30+
* derive any `blob/` prefixed capability.
2631
*/
2732
export const blob = capability({
28-
can: 'space/blob/*',
29-
/**
30-
* DID of the (memory) space where Blob is intended to
31-
* be stored.
32-
*/
33-
with: SpaceDID,
33+
can: 'blob/*',
34+
/** Storage provider DID. */
35+
with: Schema.did(),
3436
derives: equalWith,
3537
})
3638

3739
/**
38-
* Blob description for being ingested by the service.
39-
*/
40-
export const content = Schema.struct({
41-
/**
42-
* A multihash digest of the blob payload bytes, uniquely identifying blob.
43-
*/
44-
digest: Schema.bytes(),
45-
/**
46-
* Number of bytes contained by this blob. Service will provision write target
47-
* for this exact size. Attempt to write a larger Blob file will fail.
48-
*/
49-
size: Schema.integer(),
50-
})
51-
52-
/**
53-
* `space/blob/add` capability allows agent to store a Blob into a (memory) space
54-
* identified by did:key in the `with` field. Agent should compute blob multihash
55-
* and size and provide it under `nb.blob` field, allowing a service to provision
56-
* a write location for the agent to PUT desired Blob into.
40+
* The `blob/allocate` capability can be invoked to create a memory address on a
41+
* storage node where blob content can be written via a HTTP PUT request.
5742
*/
58-
export const add = capability({
59-
can: 'space/blob/add',
60-
/**
61-
* DID of the (memory) space where Blob is intended to
62-
* be stored.
63-
*/
64-
with: SpaceDID,
43+
export const allocate = capability({
44+
can: 'blob/allocate',
45+
/** Storage provider DID. */
46+
with: Schema.did(),
6547
nb: Schema.struct({
66-
/**
67-
* Blob to be added on the space.
68-
*/
48+
/** Blob to allocate. */
6949
blob: content,
50+
/** Link to the add blob task that initiated the allocation. */
51+
cause: Schema.link({ version: 1 }),
52+
/** DID of the user space where the allocation takes place. */
53+
space: SpaceDID,
7054
}),
71-
derives: equalBlob,
72-
})
73-
74-
/**
75-
* Capability can be used to remove the stored Blob from the (memory)
76-
* space identified by `with` field.
77-
*/
78-
export const remove = capability({
79-
can: 'space/blob/remove',
80-
/**
81-
* DID of the (memory) space where Blob is stored.
82-
*/
83-
with: SpaceDID,
84-
nb: Schema.struct({
85-
/**
86-
* A multihash digest of the blob payload bytes, uniquely identifying blob.
87-
*/
88-
digest: Schema.bytes(),
89-
}),
90-
derives: (claimed, delegated) => {
91-
if (claimed.with !== delegated.with) {
92-
return fail(
93-
`Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
94-
)
95-
} else if (
96-
delegated.nb.digest &&
97-
!equals(delegated.nb.digest, claimed.nb.digest)
98-
) {
99-
return fail(
100-
`Link ${
101-
claimed.nb.digest ? `${claimed.nb.digest}` : ''
102-
} violates imposed ${delegated.nb.digest} constraint.`
103-
)
104-
}
105-
return ok({})
106-
},
107-
})
108-
109-
/**
110-
* Capability can be invoked to request a list of stored Blobs in the
111-
* (memory) space identified by `with` field.
112-
*/
113-
export const list = capability({
114-
can: 'space/blob/list',
115-
/**
116-
* DID of the (memory) space where Blobs to be listed are stored.
117-
*/
118-
with: SpaceDID,
119-
nb: Schema.struct({
120-
/**
121-
* A pointer that can be moved back and forth on the list.
122-
* It can be used to paginate a list for instance.
123-
*/
124-
cursor: Schema.string().optional(),
125-
/**
126-
* Maximum number of items per page.
127-
*/
128-
size: Schema.integer().optional(),
129-
}),
130-
derives: (claimed, delegated) => {
131-
if (claimed.with !== delegated.with) {
132-
return fail(
133-
`Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
134-
)
135-
}
136-
return ok({})
137-
},
55+
derives: (claimed, delegated) =>
56+
and(equalWith(claimed, delegated)) ||
57+
and(equalBlob(claimed, delegated)) ||
58+
and(checkLink(claimed.nb.cause, delegated.nb.cause, 'cause')) ||
59+
and(equal(claimed.nb.space, delegated.nb.space, 'space')) ||
60+
ok({}),
13861
})
13962

14063
/**
141-
* Capability can be used to get the stored Blob from the (memory)
142-
* space identified by `with` field.
64+
* The `blob/accept` capability invocation should either succeed when content is
65+
* delivered on allocated address or fail if no content is allocation expires
66+
* without content being delivered.
14367
*/
144-
export const get = capability({
145-
can: 'space/blob/get/0/1',
146-
/**
147-
* DID of the (memory) space where Blob is stored.
148-
*/
149-
with: SpaceDID,
68+
export const accept = capability({
69+
can: 'blob/accept',
70+
/** Storage provider DID. */
71+
with: Schema.did(),
15072
nb: Schema.struct({
151-
/**
152-
* A multihash digest of the blob payload bytes, uniquely identifying blob.
153-
*/
154-
digest: Schema.bytes(),
73+
/** Blob to accept. */
74+
blob: content,
75+
/** DID of the user space where allocation took place. */
76+
space: SpaceDID,
77+
/** This task is blocked on `http/put` receipt available */
78+
_put: Await,
15579
}),
156-
derives: (claimed, delegated) => {
157-
if (claimed.with !== delegated.with) {
158-
return fail(
159-
`Expected 'with: "${delegated.with}"' instead got '${claimed.with}'`
160-
)
161-
} else if (
162-
delegated.nb.digest &&
163-
!equals(delegated.nb.digest, claimed.nb.digest)
164-
) {
165-
return fail(
166-
`Link ${
167-
claimed.nb.digest ? `${claimed.nb.digest}` : ''
168-
} violates imposed ${delegated.nb.digest} constraint.`
169-
)
170-
}
171-
return ok({})
172-
},
80+
derives: (claimed, delegated) =>
81+
and(equalWith(claimed, delegated)) ||
82+
and(equalBlob(claimed, delegated)) ||
83+
and(equal(claimed.nb.space, delegated.nb.space, 'space')) ||
84+
ok({}),
17385
})
17486

17587
// ⚠️ We export imports here so they are not omitted in generated typedefs
17688
// @see https://github.com/microsoft/TypeScript/issues/51548
177-
export { Schema }
89+
export { Schema, Link }

packages/capabilities/src/http.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* @module
1010
*/
1111
import { capability, Schema, ok } from '@ucanto/validator'
12-
import { content } from './blob.js'
12+
import { content } from './space/blob.js'
1313
import { equal, equalBody, equalWith, SpaceDID, Await, and } from './utils.js'
1414

1515
/**

0 commit comments

Comments
 (0)