From b0ff5c7feffd1c86d13d3d0c79a4b4015a4d03d8 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Thu, 17 Nov 2016 10:27:54 -0800 Subject: [PATCH 01/10] Refactor globals parsing and construction The construction of the global variables map passed to topdown can be re-used in a few places, so move it into topdown. The query string parsing is kept in the server package as that part is relatively simple. --- server/server.go | 52 +++++---------------------- server/server_test.go | 46 ------------------------ topdown/globals.go | 60 +++++++++++++++++++++++++++++++ topdown/globals_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 90 deletions(-) create mode 100644 topdown/globals.go create mode 100644 topdown/globals_test.go diff --git a/server/server.go b/server/server.go index 573acc5f68..3e83401dd0 100644 --- a/server/server.go +++ b/server/server.go @@ -931,10 +931,6 @@ func handleResponseJSON(w http.ResponseWriter, code int, v interface{}, pretty b handleResponse(w, code, bs) } -func globalConflictErr(k ast.Value) error { - return fmt.Errorf("conflicting global: %v: check global arguments", k) -} - func getPretty(p []string) bool { for _, x := range p { if strings.ToLower(x) == "true" { @@ -956,10 +952,12 @@ func getExplain(p []string) explainModeV1 { return explainOffV1 } -func parseGlobals(g []string) (*ast.ValueMap, error) { - globals := ast.NewValueMap() - for _, g := range g { - vs := strings.SplitN(g, ":", 2) +func parseGlobals(s []string) (*ast.ValueMap, error) { + + pairs := make([][2]*ast.Term, len(s)) + + for i := range s { + vs := strings.SplitN(s[i], ":", 2) k, err := ast.ParseTerm(vs[0]) if err != nil { return nil, err @@ -968,44 +966,10 @@ func parseGlobals(g []string) (*ast.ValueMap, error) { if err != nil { return nil, err } - switch k := k.Value.(type) { - case ast.Ref: - obj := makeTree(k[1:], v) - switch b := globals.Get(k[0].Value).(type) { - case nil: - globals.Put(k[0].Value, obj) - case ast.Object: - m, ok := b.Merge(obj) - if !ok { - return nil, globalConflictErr(k) - } - globals.Put(k[0].Value, m) - default: - return nil, globalConflictErr(k) - } - case ast.Var: - if globals.Get(k) != nil { - return nil, globalConflictErr(k) - } - globals.Put(k, v.Value) - default: - return nil, fmt.Errorf("invalid global: %v: path must be a variable or a reference", k) - } + pairs[i] = [...]*ast.Term{k, v} } - return globals, nil -} -// makeTree returns an object that represents a document where the value v is the -// leaf and elements in k represent intermediate objects. -func makeTree(k ast.Ref, v *ast.Term) ast.Object { - var obj ast.Object - for i := len(k) - 1; i >= 1; i-- { - obj = ast.Object{ast.Item(k[i], v)} - v = &ast.Term{Value: obj} - obj = ast.Object{} - } - obj = ast.Object{ast.Item(k[0], v)} - return obj + return topdown.MakeGlobals(pairs) } func renderBanner(w http.ResponseWriter) { diff --git a/server/server_test.go b/server/server_test.go index e1377a7711..19e421b135 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -642,52 +642,6 @@ func TestQueryV1Explain(t *testing.T) { } } -func TestGlobalParsing(t *testing.T) { - - tests := []struct { - note string - globals []string - expected interface{} - }{ - {"var", []string{`hello:"world"`}, `{hello: "world"}`}, - {"multiple vars", []string{`a:"a"`, `b:"b"`}, `{a: "a", b: "b"}`}, - {"multiple overlapping vars", []string{`a.b.c:"c"`, `a.b.d:"d"`, `x.y:[]`}, `{a: {"b": {"c": "c", "d": "d"}}, x: {"y": []}}`}, - {"conflicting vars", []string{`a.b:"c"`, `a.b.d:"d"`}, globalConflictErr(ast.MustParseRef("a.b.d"))}, - {"conflicting vars-2", []string{`a.b:{"c":[]}`, `a.b.c:["d"]`}, globalConflictErr(ast.MustParseRef("a.b.c"))}, - {"conflicting vars-3", []string{"a:100", `a.b:"c"`}, globalConflictErr(ast.MustParseRef("a.b"))}, - {"conflicting vars-4", []string{`a.b:"c"`, `a:100`}, globalConflictErr(ast.MustParseTerm("a").Value)}, - {"bad path", []string{`"hello":1`}, fmt.Errorf(`invalid global: "hello": path must be a variable or a reference`)}, - } - - for i, tc := range tests { - - bindings, err := parseGlobals(tc.globals) - - switch e := tc.expected.(type) { - case error: - if err == nil { - t.Errorf("%v (#%d): Expected error %v but got: %v", tc.note, i+1, e, bindings) - continue - } - if !reflect.DeepEqual(e, err) { - t.Errorf("%v (#%d): Expected error %v but got: %v", tc.note, i+1, e, err) - } - case string: - if err != nil { - t.Errorf("%v (#%d): Unexpected error: %v", tc.note, i+1, err) - continue - } - exp := ast.NewValueMap() - for _, i := range ast.MustParseTerm(e).Value.(ast.Object) { - exp.Put(i[0].Value, i[1].Value) - } - if !exp.Equal(bindings) { - t.Errorf("%v (#%d): Expected bindings to equal %v but got: %v", tc.note, i+1, exp, bindings) - } - } - } -} - const ( testMod = ` package a.b.c diff --git a/topdown/globals.go b/topdown/globals.go new file mode 100644 index 0000000000..2fa646e7d2 --- /dev/null +++ b/topdown/globals.go @@ -0,0 +1,60 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + + "github.com/open-policy-agent/opa/ast" +) + +// MakeGlobals returns a globals mapping for the given key-value pairs. +func MakeGlobals(pairs [][2]*ast.Term) (*ast.ValueMap, error) { + result := ast.NewValueMap() + for _, pair := range pairs { + k, v := pair[0], pair[1] + switch k := k.Value.(type) { + case ast.Ref: + obj := makeTree(k[1:], v) + switch b := result.Get(k[0].Value).(type) { + case nil: + result.Put(k[0].Value, obj) + case ast.Object: + m, ok := b.Merge(obj) + if !ok { + return nil, globalConflictErr(k) + } + result.Put(k[0].Value, m) + default: + return nil, globalConflictErr(k) + } + case ast.Var: + if result.Get(k) != nil { + return nil, globalConflictErr(k) + } + result.Put(k, v.Value) + default: + return nil, fmt.Errorf("invalid global: %v: path must be a variable or a reference", k) + } + } + return result, nil +} + +func globalConflictErr(k ast.Value) error { + return fmt.Errorf("conflicting global: %v: check global arguments", k) +} + +// makeTree returns an object that represents a document where the value v is the +// leaf and elements in k represent intermediate objects. +func makeTree(k ast.Ref, v *ast.Term) ast.Object { + var obj ast.Object + for i := len(k) - 1; i >= 1; i-- { + obj = ast.Object{ast.Item(k[i], v)} + v = &ast.Term{Value: obj} + obj = ast.Object{} + } + obj = ast.Object{ast.Item(k[0], v)} + return obj +} diff --git a/topdown/globals_test.go b/topdown/globals_test.go new file mode 100644 index 0000000000..8d5256bfaa --- /dev/null +++ b/topdown/globals_test.go @@ -0,0 +1,79 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "reflect" + "testing" + + "github.com/open-policy-agent/opa/ast" +) + +func TestMakeGlobals(t *testing.T) { + + tests := []struct { + note string + globals [][2]string + expected interface{} + }{ + {"var", [][2]string{{`hello`, `"world"`}}, `{hello: "world"}`}, + {"multiple vars", [][2]string{{`a`, `"a"`}, {`b`, `"b"`}}, `{a: "a", b: "b"}`}, + {"multiple overlapping vars", + [][2]string{{`a.b.c`, `"c"`}, {`a.b.d`, `"d"`}, {`x.y`, `[]`}}, + `{a: {"b": {"c": "c", "d": "d"}}, x: {"y": []}}`}, + {"conflicting vars", + [][2]string{{`a.b`, `"c"`}, {`a.b.d`, `"d"`}}, + globalConflictErr(ast.MustParseRef("a.b.d"))}, + {"conflicting vars-2", + [][2]string{{`a.b`, `{"c":[]}`}, {`a.b.c`, `["d"]`}}, + globalConflictErr(ast.MustParseRef("a.b.c"))}, + {"conflicting vars-3", + [][2]string{{"a", "100"}, {`a.b`, `"c"`}}, + globalConflictErr(ast.MustParseRef("a.b"))}, + {"conflicting vars-4", + [][2]string{{`a.b`, `"c"`}, {`a`, `100`}}, + globalConflictErr(ast.MustParseTerm("a").Value)}, + {"bad path", + [][2]string{{`"hello"`, `1`}}, + fmt.Errorf(`invalid global: "hello": path must be a variable or a reference`)}, + } + + for i, tc := range tests { + + pairs := make([][2]*ast.Term, len(tc.globals)) + + for j := range tc.globals { + k := ast.MustParseTerm(tc.globals[j][0]) + v := ast.MustParseTerm(tc.globals[j][1]) + pairs[j] = [...]*ast.Term{k, v} + } + + bindings, err := MakeGlobals(pairs) + + switch e := tc.expected.(type) { + case error: + if err == nil { + t.Errorf("%v (#%d): Expected error %v but got: %v", tc.note, i+1, e, bindings) + continue + } + if !reflect.DeepEqual(e, err) { + t.Errorf("%v (#%d): Expected error %v but got: %v", tc.note, i+1, e, err) + } + case string: + if err != nil { + t.Errorf("%v (#%d): Unexpected error: %v", tc.note, i+1, err) + continue + } + exp := ast.NewValueMap() + for _, i := range ast.MustParseTerm(e).Value.(ast.Object) { + exp.Put(i[0].Value, i[1].Value) + } + if !exp.Equal(bindings) { + t.Errorf("%v (#%d): Expected bindings to equal %v but got: %v", tc.note, i+1, exp, bindings) + } + } + } +} From 389d131dd5635d8c6f2fdd61d2e6d9e95f853005 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Wed, 16 Nov 2016 15:28:55 -0800 Subject: [PATCH 02/10] Add global variable support to REPL --- repl/repl.go | 60 ++++++++++++++++++++++++++++++++++++----- repl/repl_test.go | 21 +++++++++++++++ topdown/globals_test.go | 3 +++ 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/repl/repl.go b/repl/repl.go index 856bbca3cc..e0c9e67e4c 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -451,6 +451,46 @@ func (r *REPL) loadCompiler() (*ast.Compiler, error) { return compiler, nil } +// loadGlobals returns the globals mapping currently defined in the REPL. The +// REPL loads globals from the data.repl.globals document. +func (r *REPL) loadGlobals(compiler *ast.Compiler) (*ast.ValueMap, error) { + + params := topdown.NewQueryParams(compiler, r.store, r.txn, nil, []interface{}{"repl", "globals"}) + + result, err := topdown.Query(params) + if err != nil { + return nil, err + } + + if _, ok := result.(topdown.Undefined); ok { + return nil, nil + } + + pairs := [][2]*ast.Term{} + + obj, ok := result.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("globals is %T but expected object", result) + } + + for k, v := range obj { + + gk, err := ast.ParseTerm(k) + if err != nil { + return nil, err + } + + gv, err := ast.InterfaceToValue(v) + if err != nil { + return nil, err + } + + pairs = append(pairs, [...]*ast.Term{gk, &ast.Term{Value: gv}}) + } + + return topdown.MakeGlobals(pairs) +} + func (r *REPL) evalStatement(stmt interface{}) bool { switch s := stmt.(type) { case ast.Body: @@ -470,7 +510,12 @@ func (r *REPL) evalStatement(stmt interface{}) bool { fmt.Fprintln(r.output, "error:", err) return false } - return r.evalBody(compiler, body) + globals, err := r.loadGlobals(compiler) + if err != nil { + fmt.Fprintln(r.output, "error:", err) + return false + } + return r.evalBody(compiler, globals, body) case *ast.Rule: if err := r.compileRule(s); err != nil { fmt.Fprintln(r.output, "error:", err) @@ -483,7 +528,7 @@ func (r *REPL) evalStatement(stmt interface{}) bool { return false } -func (r *REPL) evalBody(compiler *ast.Compiler, body ast.Body) bool { +func (r *REPL) evalBody(compiler *ast.Compiler, globals *ast.ValueMap, body ast.Body) bool { // Special case for positive, single term inputs. if len(body) == 1 { @@ -491,14 +536,15 @@ func (r *REPL) evalBody(compiler *ast.Compiler, body ast.Body) bool { if !expr.Negated { if _, ok := expr.Terms.(*ast.Term); ok { if singleValue(body) { - return r.evalTermSingleValue(compiler, body) + return r.evalTermSingleValue(compiler, globals, body) } - return r.evalTermMultiValue(compiler, body) + return r.evalTermMultiValue(compiler, globals, body) } } } ctx := topdown.NewContext(body, compiler, r.store, r.txn) + ctx.Globals = globals var buf *topdown.BufferTracer @@ -610,13 +656,14 @@ func (r *REPL) evalPackage(p *ast.Package) bool { // and comprehensions always evaluate to a single value. To handle references, this function // still executes the query, except it does so by rewriting the body to assign the term // to a variable. This allows the REPL to obtain the result even if the term is false. -func (r *REPL) evalTermSingleValue(compiler *ast.Compiler, body ast.Body) bool { +func (r *REPL) evalTermSingleValue(compiler *ast.Compiler, globals *ast.ValueMap, body ast.Body) bool { term := body[0].Terms.(*ast.Term) outputVar := ast.Wildcard body = ast.NewBody(ast.Equality.Expr(term, outputVar)) ctx := topdown.NewContext(body, compiler, r.store, r.txn) + ctx.Globals = globals var buf *topdown.BufferTracer @@ -656,7 +703,7 @@ func (r *REPL) evalTermSingleValue(compiler *ast.Compiler, body ast.Body) bool { // evalTermMultiValue evaluates and prints terms in cases where the term may evaluate to multiple // ground values, e.g., a[i], [servers[x]], etc. -func (r *REPL) evalTermMultiValue(compiler *ast.Compiler, body ast.Body) bool { +func (r *REPL) evalTermMultiValue(compiler *ast.Compiler, globals *ast.ValueMap, body ast.Body) bool { // Mangle the expression in the same way we do for evalTermSingleValue. When handling the // evaluation result below, we will ignore this variable. @@ -665,6 +712,7 @@ func (r *REPL) evalTermMultiValue(compiler *ast.Compiler, body ast.Body) bool { body = ast.NewBody(ast.Equality.Expr(term, outputVar)) ctx := topdown.NewContext(body, compiler, r.store, r.txn) + ctx.Globals = globals var buf *topdown.BufferTracer diff --git a/repl/repl_test.go b/repl/repl_test.go index 456130b1ab..6bd765f870 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -522,6 +522,27 @@ func TestEvalBodyContainingWildCards(t *testing.T) { } +func TestEvalBodyGlobals(t *testing.T) { + store := newTestStore() + var buffer bytes.Buffer + repl := newRepl(store, &buffer) + + repl.OneShot("package repl") + repl.OneShot(`globals["foo.bar"] = "hello" :- true`) + repl.OneShot(`globals["baz"] = data.a[0].b.c[2] :- true`) + repl.OneShot("package test") + repl.OneShot("import foo.bar") + repl.OneShot("import baz") + repl.OneShot(`p :- bar = "hello", baz = false`) + + repl.OneShot("p") + + result := buffer.String() + if result != "true\n" { + t.Fatalf("expected true but got: %v", result) + } +} + func TestEvalImport(t *testing.T) { store := newTestStore() var buffer bytes.Buffer diff --git a/topdown/globals_test.go b/topdown/globals_test.go index 8d5256bfaa..24b4fd270e 100644 --- a/topdown/globals_test.go +++ b/topdown/globals_test.go @@ -24,6 +24,9 @@ func TestMakeGlobals(t *testing.T) { {"multiple overlapping vars", [][2]string{{`a.b.c`, `"c"`}, {`a.b.d`, `"d"`}, {`x.y`, `[]`}}, `{a: {"b": {"c": "c", "d": "d"}}, x: {"y": []}}`}, + {"ref value", + [][2]string{{"foo.bar", "data.com.example.widgets[i]"}}, + `{foo: {"bar": data.com.example.widgets[i]}}`}, {"conflicting vars", [][2]string{{`a.b`, `"c"`}, {`a.b.d`, `"d"`}}, globalConflictErr(ast.MustParseRef("a.b.d"))}, From bc25e3026e69fc2013070c312b75e96667a02908 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Thu, 17 Nov 2016 14:44:44 -0800 Subject: [PATCH 03/10] Add constructors for Term and Expr --- ast/policy.go | 9 +++++++++ ast/term.go | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/ast/policy.go b/ast/policy.go index 96a3442d8f..1789ec3679 100644 --- a/ast/policy.go +++ b/ast/policy.go @@ -370,6 +370,15 @@ func (body Body) Vars(skipClosures bool) VarSet { return vis.vars } +// NewExpr returns a new Expr object. +func NewExpr(terms interface{}) *Expr { + return &Expr{ + Negated: false, + Terms: terms, + Index: 0, + } +} + // Complement returns a copy of this expression with the negation flag flipped. func (expr *Expr) Complement() *Expr { cpy := *expr diff --git a/ast/term.go b/ast/term.go index 6235b532a5..a39eb2d734 100644 --- a/ast/term.go +++ b/ast/term.go @@ -122,6 +122,13 @@ type Term struct { Location *Location `json:"-"` // the location of the Term in the source } +// NewTerm returns a new Term object. +func NewTerm(v Value) *Term { + return &Term{ + Value: v, + } +} + // Equal returns true if this term equals the other term. Equality is // defined for each kind of term. func (term *Term) Equal(other *Term) bool { From 20f62cb9006a5a4595e8d8f0b24bf1006c73d025 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Thu, 17 Nov 2016 17:53:26 -0800 Subject: [PATCH 04/10] Refactor topdown.QueryParams to use ref for Path --- repl/repl.go | 2 +- server/server.go | 28 ++++++++------------------ test/scheduler/scheduler_bench_test.go | 2 +- test/scheduler/scheduler_test.go | 2 +- topdown/example_test.go | 2 +- topdown/explain/explain_test.go | 2 +- topdown/topdown.go | 22 +++----------------- topdown/topdown_test.go | 9 ++++++--- topdown/trace_test.go | 2 +- 9 files changed, 23 insertions(+), 48 deletions(-) diff --git a/repl/repl.go b/repl/repl.go index e0c9e67e4c..93cefeed5a 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -455,7 +455,7 @@ func (r *REPL) loadCompiler() (*ast.Compiler, error) { // REPL loads globals from the data.repl.globals document. func (r *REPL) loadGlobals(compiler *ast.Compiler) (*ast.ValueMap, error) { - params := topdown.NewQueryParams(compiler, r.store, r.txn, nil, []interface{}{"repl", "globals"}) + params := topdown.NewQueryParams(compiler, r.store, r.txn, nil, ast.MustParseRef("data.repl.globals")) result, err := topdown.Query(params) if err != nil { diff --git a/server/server.go b/server/server.go index 3e83401dd0..a8c0053159 100644 --- a/server/server.go +++ b/server/server.go @@ -401,7 +401,7 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { // Gather request parameters. vars := mux.Vars(r) - path := stringPathToInterface(vars["path"]) + path := stringPathToDataRef(vars["path"]) pretty := getPretty(r.URL.Query()["pretty"]) explainMode := getExplain(r.URL.Query()["explain"]) globals, err := parseGlobals(r.URL.Query()["global"]) @@ -463,9 +463,7 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { func (s *Server) v1DataPatch(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - - root := ast.Ref{ast.DefaultRootDocument} - root = append(root, stringPathToRef(vars["path"])...) + root := stringPathToDataRef(vars["path"]) ops := []patchV1{} if err := json.NewDecoder(r.Body).Decode(&ops); err != nil { @@ -819,6 +817,12 @@ func (s *Server) writeConflict(op storage.PatchOp, path ast.Ref) error { return nil } +func stringPathToDataRef(s string) (r ast.Ref) { + result := ast.Ref{ast.DefaultRootDocument} + result = append(result, stringPathToRef(s)...) + return result +} + func stringPathToRef(s string) (r ast.Ref) { if len(s) == 0 { return r @@ -838,22 +842,6 @@ func stringPathToRef(s string) (r ast.Ref) { return r } -func stringPathToInterface(s string) (r []interface{}) { - if len(s) == 0 { - return r - } - p := strings.Split(s, "/") - for _, x := range p { - i, err := strconv.Atoi(x) - if err != nil { - r = append(r, x) - } else { - r = append(r, float64(i)) - } - } - return r -} - func handleError(w http.ResponseWriter, code int, err error) { handleErrorf(w, code, err.Error()) } diff --git a/test/scheduler/scheduler_bench_test.go b/test/scheduler/scheduler_bench_test.go index acce939a99..39504d2a0a 100644 --- a/test/scheduler/scheduler_bench_test.go +++ b/test/scheduler/scheduler_bench_test.go @@ -68,7 +68,7 @@ func setupBenchmark(nodes int, pods int) *topdown.QueryParams { globals := ast.NewValueMap() req := ast.MustParseTerm(requestedPod).Value globals.Put(ast.Var("requested_pod"), req) - path := []interface{}{"opa", "test", "scheduler", "fit"} + path := ast.MustParseRef("data.opa.test.scheduler.fit") txn := storage.NewTransactionOrDie(store) params := topdown.NewQueryParams(c, store, txn, globals, path) diff --git a/test/scheduler/scheduler_test.go b/test/scheduler/scheduler_test.go index 236c8a1143..2825ff2870 100644 --- a/test/scheduler/scheduler_test.go +++ b/test/scheduler/scheduler_test.go @@ -57,7 +57,7 @@ func setup(t *testing.T, filename string) *topdown.QueryParams { globals := ast.NewValueMap() req := ast.MustParseTerm(requestedPod).Value globals.Put(ast.Var("requested_pod"), req) - path := []interface{}{"opa", "test", "scheduler", "fit"} + path := ast.MustParseRef("data.opa.test.scheduler.fit") txn := storage.NewTransactionOrDie(store) params := topdown.NewQueryParams(c, store, txn, globals, path) diff --git a/topdown/example_test.go b/topdown/example_test.go index 691b02b3b9..883981e18c 100644 --- a/topdown/example_test.go +++ b/topdown/example_test.go @@ -105,7 +105,7 @@ func ExampleQuery() { // accept additional documents (which are referred to as "globals"). In this case we have no // additional documents. globals := ast.NewValueMap() - params := topdown.NewQueryParams(compiler, store, txn, globals, []interface{}{"opa", "example", "p"}) + params := topdown.NewQueryParams(compiler, store, txn, globals, ast.MustParseRef("data.opa.example.p")) // Execute the query against "p". v1, err1 := topdown.Query(params) diff --git a/topdown/explain/explain_test.go b/topdown/explain/explain_test.go index 0bcae82d50..8650352132 100644 --- a/topdown/explain/explain_test.go +++ b/topdown/explain/explain_test.go @@ -200,7 +200,7 @@ func executeQuery(data string, compiler *ast.Compiler, tracer topdown.Tracer) { txn := storage.NewTransactionOrDie(store) defer store.Close(txn) - params := topdown.NewQueryParams(compiler, store, txn, nil, []interface{}{"test", "p"}) + params := topdown.NewQueryParams(compiler, store, txn, nil, ast.MustParseRef("data.test.p")) params.Tracer = tracer _, err := topdown.Query(params) diff --git a/topdown/topdown.go b/topdown/topdown.go index 914ec0ff65..2f483d623c 100644 --- a/topdown/topdown.go +++ b/topdown/topdown.go @@ -520,11 +520,11 @@ type QueryParams struct { Transaction storage.Transaction Globals *ast.ValueMap Tracer Tracer - Path []interface{} + Path ast.Ref } // NewQueryParams returns a new QueryParams. -func NewQueryParams(compiler *ast.Compiler, store *storage.Storage, txn storage.Transaction, globals *ast.ValueMap, path []interface{}) *QueryParams { +func NewQueryParams(compiler *ast.Compiler, store *storage.Storage, txn storage.Transaction, globals *ast.ValueMap, path ast.Ref) *QueryParams { return &QueryParams{ Compiler: compiler, Store: store, @@ -545,24 +545,8 @@ func (q *QueryParams) NewContext(body ast.Body) *Context { // Query returns the document identified by the path. func Query(params *QueryParams) (interface{}, error) { - ref := ast.Ref{ast.DefaultRootDocument} - for _, v := range params.Path { - switch v := v.(type) { - case float64: - ref = append(ref, ast.NumberTerm(v)) - case string: - ref = append(ref, ast.StringTerm(v)) - case bool: - ref = append(ref, ast.BooleanTerm(v)) - case nil: - ref = append(ref, ast.NullTerm()) - default: - return nil, fmt.Errorf("bad path element: %v (%T)", v, v) - } - } - // Construct and execute a query to obtain the value for the reference. - query := ast.NewBody(ast.Equality.Expr(ast.RefTerm(ref...), ast.Wildcard)) + query := ast.NewBody(ast.Equality.Expr(ast.RefTerm(params.Path...), ast.Wildcard)) ctx := params.NewContext(query) var result interface{} = Undefined{} var err error diff --git a/topdown/topdown_test.go b/topdown/topdown_test.go index f374f67923..2f3f18679b 100644 --- a/topdown/topdown_test.go +++ b/topdown/topdown_test.go @@ -1604,7 +1604,7 @@ func runTopDownTracingTestCase(t *testing.T, module string, n int, cases map[int store := storage.New(storage.InMemoryWithJSONConfig(data)) txn := storage.NewTransactionOrDie(store) - params := NewQueryParams(compiler, store, txn, nil, []interface{}{"test", "p"}) + params := NewQueryParams(compiler, store, txn, nil, ast.MustParseRef("data.test.p")) buf := NewBufferTracer() params.Tracer = buf @@ -1653,10 +1653,13 @@ func assertTopDown(t *testing.T, compiler *ast.Compiler, store *storage.Storage, txn := storage.NewTransactionOrDie(store) defer store.Close(txn) + ref := ast.MustParseRef("data." + strings.Join(path, ".")) + params := NewQueryParams(compiler, store, txn, g, ref) + testutil.Subtest(t, note, func(t *testing.T) { switch e := expected.(type) { case error: - result, err := Query(&QueryParams{Compiler: compiler, Store: store, Transaction: txn, Path: p, Globals: g}) + result, err := Query(params) if err == nil { t.Errorf("Expected error but got: %v", result) return @@ -1666,7 +1669,7 @@ func assertTopDown(t *testing.T, compiler *ast.Compiler, store *storage.Storage, } case string: expected := loadExpectedSortedResult(e) - result, err := Query(&QueryParams{Compiler: compiler, Store: store, Transaction: txn, Path: p, Globals: g}) + result, err := Query(params) if err != nil { t.Errorf("Unexpected error: %v", err) return diff --git a/topdown/trace_test.go b/topdown/trace_test.go index 52d73bf100..640728a3dc 100644 --- a/topdown/trace_test.go +++ b/topdown/trace_test.go @@ -61,7 +61,7 @@ func TestPrettyTrace(t *testing.T) { store := storage.New(storage.InMemoryWithJSONConfig(data)) txn := storage.NewTransactionOrDie(store) - params := NewQueryParams(compiler, store, txn, nil, []interface{}{"test", "p"}) + params := NewQueryParams(compiler, store, txn, nil, ast.MustParseRef("data.test.p")) tracer := NewBufferTracer() params.Tracer = tracer From b45daedf84da1a0a9f71ec432d2b9e097d69c97f Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Fri, 18 Nov 2016 10:41:05 -0800 Subject: [PATCH 05/10] Refactor ValueToInterface and related helpers These functions do not require the entire context. They only depend on the ability to resolve references to base documents to native Go values. As such, it makes sense to limit the coupling. --- topdown/topdown.go | 61 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/topdown/topdown.go b/topdown/topdown.go index 2f483d623c..76f77d33c4 100644 --- a/topdown/topdown.go +++ b/topdown/topdown.go @@ -139,6 +139,11 @@ func (ctx *Context) Current() *ast.Expr { return ctx.Query[ctx.Index] } +// Resolve returns the native Go value referred to by the ref. +func (ctx *Context) Resolve(ref ast.Ref) (interface{}, error) { + return ctx.Store.Read(ctx.txn, ref) +} + // Step returns a new context to evaluate the next expression. func (ctx *Context) Step() *Context { cpy := *ctx @@ -572,8 +577,14 @@ func (undefined Undefined) String() string { return "" } -// ResolveRefs returns an AST value obtained by replacing references in v with -// values from storage. +// Resolver defines the interface for resolving references to base documents to +// native Go values. The native Go value types map to JSON types. +type Resolver interface { + Resolve(ref ast.Ref) (value interface{}, err error) +} + +// ResolveRefs returns the AST value obtained by resolving references to base +// doccuments. func ResolveRefs(v ast.Value, ctx *Context) (ast.Value, error) { result, err := ast.TransformRefs(v, func(r ast.Ref) (ast.Value, error) { return lookupValue(ctx, r) @@ -587,10 +598,8 @@ func ResolveRefs(v ast.Value, ctx *Context) (ast.Value, error) { // ValueToInterface returns the underlying Go value associated with an AST value. // If the value is a reference, the reference is fetched from storage. Composite // AST values such as objects and arrays are converted recursively. -func ValueToInterface(v ast.Value, ctx *Context) (interface{}, error) { - +func ValueToInterface(v ast.Value, resolver Resolver) (interface{}, error) { switch v := v.(type) { - case ast.Null: return nil, nil case ast.Boolean: @@ -599,22 +608,20 @@ func ValueToInterface(v ast.Value, ctx *Context) (interface{}, error) { return float64(v), nil case ast.String: return string(v), nil - case ast.Array: buf := []interface{}{} for _, x := range v { - x1, err := ValueToInterface(x.Value, ctx) + x1, err := ValueToInterface(x.Value, resolver) if err != nil { return nil, err } buf = append(buf, x1) } return buf, nil - case ast.Object: buf := map[string]interface{}{} for _, x := range v { - k, err := ValueToInterface(x[0].Value, ctx) + k, err := ValueToInterface(x[0].Value, resolver) if err != nil { return nil, err } @@ -622,42 +629,34 @@ func ValueToInterface(v ast.Value, ctx *Context) (interface{}, error) { if !stringKey { return nil, fmt.Errorf("object key type %T", k) } - v, err := ValueToInterface(x[1].Value, ctx) + v, err := ValueToInterface(x[1].Value, resolver) if err != nil { return nil, err } buf[asStr] = v } return buf, nil - case *ast.Set: buf := []interface{}{} for _, x := range *v { - x1, err := ValueToInterface(x.Value, ctx) + x1, err := ValueToInterface(x.Value, resolver) if err != nil { return nil, err } buf = append(buf, x1) } return buf, nil - - // References convert to native values via lookup. case ast.Ref: - return ctx.Store.Read(ctx.txn, v) - + return resolver.Resolve(v) default: - v = PlugValue(v, ctx.Binding) - if !v.IsGround() { - return nil, fmt.Errorf("unbound value: %v", v) - } - return ValueToInterface(v, ctx) + return nil, fmt.Errorf("unbound value: %v", v) } } // ValueToSlice returns the underlying Go value associated with an AST value. // If the value is a reference, the reference is fetched from storage. -func ValueToSlice(v ast.Value, ctx *Context) ([]interface{}, error) { - x, err := ValueToInterface(v, ctx) +func ValueToSlice(v ast.Value, resolver Resolver) ([]interface{}, error) { + x, err := ValueToInterface(v, resolver) if err != nil { return nil, err } @@ -670,8 +669,8 @@ func ValueToSlice(v ast.Value, ctx *Context) ([]interface{}, error) { // ValueToFloat64 returns the underlying Go value associated with an AST value. // If the value is a reference, the reference is fetched from storage. -func ValueToFloat64(v ast.Value, ctx *Context) (float64, error) { - x, err := ValueToInterface(v, ctx) +func ValueToFloat64(v ast.Value, resolver Resolver) (float64, error) { + x, err := ValueToInterface(v, resolver) if err != nil { return 0, err } @@ -684,8 +683,8 @@ func ValueToFloat64(v ast.Value, ctx *Context) (float64, error) { // ValueToInt returns the underlying Go value associated with an AST value. // If the value is a reference, the reference is fetched from storage. -func ValueToInt(v ast.Value, ctx *Context) (int64, error) { - x, err := ValueToFloat64(v, ctx) +func ValueToInt(v ast.Value, resolver Resolver) (int64, error) { + x, err := ValueToFloat64(v, resolver) if err != nil { return 0, err } @@ -697,8 +696,8 @@ func ValueToInt(v ast.Value, ctx *Context) (int64, error) { // ValueToString returns the underlying Go value associated with an AST value. // If the value is a reference, the reference is fetched from storage. -func ValueToString(v ast.Value, ctx *Context) (string, error) { - x, err := ValueToInterface(v, ctx) +func ValueToString(v ast.Value, resolver Resolver) (string, error) { + x, err := ValueToInterface(v, resolver) if err != nil { return "", err } @@ -710,8 +709,8 @@ func ValueToString(v ast.Value, ctx *Context) (string, error) { } // ValueToStrings returns a slice of strings associated with an AST value. -func ValueToStrings(v ast.Value, ctx *Context) ([]string, error) { - sl, err := ValueToSlice(v, ctx) +func ValueToStrings(v ast.Value, resolver Resolver) ([]string, error) { + sl, err := ValueToSlice(v, resolver) if err != nil { return nil, err } From 6fa4848c7a5e06aab25c882a9947eb807ccef98c Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Fri, 18 Nov 2016 12:35:27 -0800 Subject: [PATCH 06/10] Update topdown.Query to support non-ground globals This change required refactoring the topdown.Query interface to return multiple document values instead of a single one. --- repl/repl.go | 4 +- server/server.go | 7 +- test/scheduler/scheduler_bench_test.go | 8 +- test/scheduler/scheduler_test.go | 8 +- topdown/example_test.go | 2 +- topdown/topdown.go | 128 +++++++++++++++++++++++-- topdown/topdown_test.go | 97 ++++++++++++++++--- 7 files changed, 217 insertions(+), 37 deletions(-) diff --git a/repl/repl.go b/repl/repl.go index 93cefeed5a..14f52eaad2 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -462,13 +462,13 @@ func (r *REPL) loadGlobals(compiler *ast.Compiler) (*ast.ValueMap, error) { return nil, err } - if _, ok := result.(topdown.Undefined); ok { + if result.Undefined() { return nil, nil } pairs := [][2]*ast.Term{} - obj, ok := result.(map[string]interface{}) + obj, ok := result[0].Result.(map[string]interface{}) if !ok { return nil, fmt.Errorf("globals is %T but expected object", result) } diff --git a/server/server.go b/server/server.go index a8c0053159..92bc90c181 100644 --- a/server/server.go +++ b/server/server.go @@ -429,7 +429,7 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { } // Execute query. - result, err := topdown.Query(params) + qrs, err := topdown.Query(params) // Handle results. if err != nil { @@ -437,7 +437,7 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { return } - if _, ok := result.(topdown.Undefined); ok { + if qrs.Undefined() { if explainMode == explainFullV1 { handleResponseJSON(w, 404, newTraceV1(*buf), pretty) } else { @@ -446,6 +446,9 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { return } + // TODO(tsandall): update to handle non-ground global case + result := qrs[0].Result + switch explainMode { case explainOffV1: handleResponseJSON(w, 200, result, pretty) diff --git a/test/scheduler/scheduler_bench_test.go b/test/scheduler/scheduler_bench_test.go index 39504d2a0a..6ae3d49a3b 100644 --- a/test/scheduler/scheduler_bench_test.go +++ b/test/scheduler/scheduler_bench_test.go @@ -33,17 +33,17 @@ func runSchedulerBenchmark(b *testing.B, nodes int, pods int) { defer params.Store.Close(params.Transaction) b.ResetTimer() for i := 0; i < b.N; i++ { - r, err := topdown.Query(params) + qrs, err := topdown.Query(params) if err != nil { b.Fatal("unexpected error:", err) } - ws := r.(map[string]interface{}) + ws := qrs[0].Result.(map[string]interface{}) if len(ws) != nodes { - b.Fatal("unexpected query result:", r) + b.Fatal("unexpected query result:", qrs) } for n, w := range ws { if fmt.Sprintf("%.3f", w) != "5.014" { - b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) + b.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, qrs) } } } diff --git a/test/scheduler/scheduler_test.go b/test/scheduler/scheduler_test.go index 2825ff2870..b01455e7bb 100644 --- a/test/scheduler/scheduler_test.go +++ b/test/scheduler/scheduler_test.go @@ -20,18 +20,18 @@ func TestScheduler(t *testing.T) { params := setup(t, "data_10nodes_30pods.json") defer params.Store.Close(params.Transaction) - r, err := topdown.Query(params) + qrs, err := topdown.Query(params) if err != nil { t.Fatal("unexpected error:", err) } - ws := r.(map[string]interface{}) + ws := qrs[0].Result.(map[string]interface{}) if len(ws) != 10 { - t.Fatal("unexpected query result:", r) + t.Fatal("unexpected query result:", qrs) } for n, w := range ws { if fmt.Sprintf("%.3f", w) != "5.014" { - t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, r) + t.Fatalf("unexpected weight for: %v: %v\n\nDumping all weights:\n\n%v\n", n, w, qrs) } } } diff --git a/topdown/example_test.go b/topdown/example_test.go index 883981e18c..f7dd506b57 100644 --- a/topdown/example_test.go +++ b/topdown/example_test.go @@ -111,7 +111,7 @@ func ExampleQuery() { v1, err1 := topdown.Query(params) // Inspect the result. - fmt.Println("v1:", v1) + fmt.Println("v1:", v1[0].Result) fmt.Println("err1:", err1) // Output: diff --git a/topdown/topdown.go b/topdown/topdown.go index 76f77d33c4..96826713a1 100644 --- a/topdown/topdown.go +++ b/topdown/topdown.go @@ -547,13 +547,49 @@ func (q *QueryParams) NewContext(body ast.Body) *Context { return ctx } -// Query returns the document identified by the path. -func Query(params *QueryParams) (interface{}, error) { +// QueryResult represents a single query result. +type QueryResult struct { + Result interface{} // Result contains the document referred to by the params Path. + Globals map[string]interface{} // Globals contains bindings for variables in the params Globals. +} + +func (qr *QueryResult) String() string { + return fmt.Sprintf("[%v %v]", qr.Result, qr.Globals) +} + +// QueryResultSet represents a collection of query results. +type QueryResultSet []*QueryResult + +// Undefined returns true if the query did not find any results. +func (qrs QueryResultSet) Undefined() bool { + return len(qrs) == 0 +} + +// Add inserts a result into the query result set. +func (qrs *QueryResultSet) Add(qr *QueryResult) { + *qrs = append(*qrs, qr) +} + +// Query returns the value of document referred to by the params Path field. If +// the params Globals field contains values that are non-ground (i.e., they +// contain variables), then the result may contain multiple entries. +func Query(params *QueryParams) (QueryResultSet, error) { + + if params.Globals.Len() == 0 { + return queryOne(params) + } + + return queryN(params) +} + +// queryOne returns a QueryResultSet containing the value of the document +// referred to by the params Path field. If the document is not defined, nil is +// returned. +func queryOne(params *QueryParams) (QueryResultSet, error) { - // Construct and execute a query to obtain the value for the reference. query := ast.NewBody(ast.Equality.Expr(ast.RefTerm(params.Path...), ast.Wildcard)) ctx := params.NewContext(query) - var result interface{} = Undefined{} + var result interface{} = struct{}{} var err error err = Eval(ctx, func(ctx *Context) error { @@ -566,15 +602,78 @@ func Query(params *QueryParams) (interface{}, error) { return nil, err } - return result, nil + if _, ok := result.(struct{}); ok { + return nil, nil + } + + return QueryResultSet{&QueryResult{result, nil}}, nil } -// Undefined represents the absence of bindings that satisfy a completely defined rule. -// See the documentation for Query for more details. -type Undefined struct{} +// queryN returns a QueryResultSet containing the values of the document +// referred to by the params Path field. There may be zero or more values +// depending on the values of the params Globals field. +// +// For example, if the globals refer to one or more undefined documents, the set +// will be empty. On the other hand, if the globals contain non-ground +// references where there are multiple valid sets of bindings, the result set +// may contain multiple values. +func queryN(params *QueryParams) (QueryResultSet, error) { + + qrs := QueryResultSet{} + vars := ast.NewVarSet() + resolver := resolver{params.Store, params.Transaction} -func (undefined Undefined) String() string { - return "" + params.Globals.Iter(func(_, v ast.Value) bool { + ast.WalkRefs(v, func(r ast.Ref) bool { + vars.Update(r.OutputVars()) + return false + }) + return false + }) + + err := evalGlobals(params, func(globals *ast.ValueMap, root *Context) error { + params.Globals = globals + result, err := queryOne(params) + if err != nil || result.Undefined() { + return err + } + + bindings := map[string]interface{}{} + for v := range vars { + binding, err := ValueToInterface(PlugValue(v, root.Binding), resolver) + if err != nil { + return err + } + bindings[v.String()] = binding + } + + qrs.Add(&QueryResult{result[0].Result, bindings}) + return nil + }) + + return qrs, err +} + +// evalGlobals constructs query to find bindings for all variables in the params +// Globals field. +func evalGlobals(params *QueryParams, iter func(*ast.ValueMap, *Context) error) error { + exprs := []*ast.Expr{} + params.Globals.Iter(func(k, v ast.Value) bool { + exprs = append(exprs, ast.Equality.Expr(ast.NewTerm(k), ast.NewTerm(v))) + return false + }) + + query := ast.NewBody(exprs...) + ctx := params.NewContext(query) + + return Eval(ctx, func(ctx *Context) error { + globals := ast.NewValueMap() + params.Globals.Iter(func(k, _ ast.Value) bool { + globals.Put(k, PlugValue(k, ctx.Binding)) + return false + }) + return iter(globals, ctx) + }) } // Resolver defines the interface for resolving references to base documents to @@ -583,6 +682,15 @@ type Resolver interface { Resolve(ref ast.Ref) (value interface{}, err error) } +type resolver struct { + store *storage.Storage + txn storage.Transaction +} + +func (r resolver) Resolve(ref ast.Ref) (interface{}, error) { + return r.store.Read(r.txn, ref) +} + // ResolveRefs returns the AST value obtained by resolving references to base // doccuments. func ResolveRefs(v ast.Value, ctx *Context) (ast.Value, error) { diff --git a/topdown/topdown_test.go b/topdown/topdown_test.go index 2f3f18679b..8f25f2091c 100644 --- a/topdown/topdown_test.go +++ b/topdown/topdown_test.go @@ -1101,7 +1101,12 @@ func TestTopDownGlobalVars(t *testing.T) { q[x] :- req1.foo = x, req2as.bar = x, r[x] r[x] :- {"foo": req2as.bar, "bar": [x]} = {"foo": x, "bar": [req1.foo]} s :- b.x[0] = 1 - t :- req4as.x[0] = 1`}) + t :- req4as.x[0] = 1 + u[x] :- b[_] = x, x > 1 + w = [[1,2], [3,4]] + gt1 :- req1 > 1 + keys[x] = y :- data.numbers[_] = x, to_number(x, y) + `}) store := storage.New(storage.InMemoryWithJSONConfig(loadSmallTestData())) @@ -1133,6 +1138,46 @@ func TestTopDownGlobalVars(t *testing.T) { } } }`, "true") + + assertTopDown(t, compiler, store, "global vars (embedded ref to base doc)", []string{"z", "s"}, `{ + req3: { + "a": { + "b": { + "x": data.a + } + } + } + }`, "true") + + assertTopDown(t, compiler, store, "global vars (embedded non-ground ref to base doc)", []string{"z", "u"}, `{ + req3: { + "a": { + "b": data.l[x].c + } + } + }`, [][2]string{ + {"[2,3,4]", `{"x": 0}`}, + {"[2,3,4,5]", `{"x": 1}`}, + }) + + assertTopDown(t, compiler, store, "global vars (embedded non-ground ref to virtual doc)", []string{"z", "u"}, `{ + req3: { + "a": { + "b": data.z.w[x] + } + } + }`, [][2]string{ + {"[2]", `{"x": 0}`}, + {"[3,4]", `{"x": 1}`}, + }) + + assertTopDown(t, compiler, store, "global vars (non-ground ref to virtual doc-2)", []string{"z", "gt1"}, `{ + req1: data.z.keys[x] + }`, [][2]string{ + {"true", `{"x": "2"}`}, + {"true", `{"x": "3"}`}, + {"true", `{"x": "4"}`}, + }) } func TestTopDownCaching(t *testing.T) { @@ -1494,10 +1539,7 @@ func loadExpectedBindings(input string) []*ast.ValueMap { return expected } -func loadExpectedResult(input string) interface{} { - if len(input) == 0 { - return Undefined{} - } +func parseJSON(input string) interface{} { var data interface{} if err := json.Unmarshal([]byte(input), &data); err != nil { panic(err) @@ -1505,8 +1547,15 @@ func loadExpectedResult(input string) interface{} { return data } -func loadExpectedSortedResult(input string) interface{} { - data := loadExpectedResult(input) +func parseQueryResultSetJSON(input [][2]string) (result QueryResultSet) { + for i := range input { + result.Add(&QueryResult{parseJSON(input[i][0]), parseJSON(input[i][1]).(map[string]interface{})}) + } + return result +} + +func parseSortedJSON(input string) interface{} { + data := parseJSON(input) switch data := data.(type) { case []interface{}: sort.Sort(resultSet(data)) @@ -1667,23 +1716,43 @@ func assertTopDown(t *testing.T, compiler *ast.Compiler, store *storage.Storage, if err.Error() != e.Error() { t.Errorf("Expected error %v but got: %v", e, err) } + + case [][2]string: + qrs, err := Query(params) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + expected := parseQueryResultSetJSON(e) + + if !reflect.DeepEqual(expected, qrs) { + t.Fatalf("Expected %v but got: %v", expected, qrs) + } + case string: - expected := loadExpectedSortedResult(e) - result, err := Query(params) + qrs, err := Query(params) + if err != nil { - t.Errorf("Unexpected error: %v", err) + t.Fatalf("Unexpected error: %v", err) + } + + if len(e) == 0 { + if !qrs.Undefined() { + t.Fatalf("Expected undefined result but got: %v", qrs) + } return } - p := ast.MustParseRef(fmt.Sprintf("data.%v", strings.Join(path, "."))) + expected := parseSortedJSON(e) // Sort set results so that comparisons are not dependant on order. + p := ast.MustParseRef(fmt.Sprintf("data.%v", strings.Join(path, "."))) if rs := compiler.GetRulesExact(p); len(rs) > 0 && rs[0].DocKind() == ast.PartialSetDoc { - sort.Sort(resultSet(result.([]interface{}))) + sort.Sort(resultSet(qrs[0].Result.([]interface{}))) } - if !reflect.DeepEqual(result, expected) { - t.Errorf("Expected %v but got: %v", expected, result) + if !reflect.DeepEqual(qrs[0].Result, expected) { + t.Errorf("Expected %v but got: %v", expected, qrs[0].Result) } } }) From cf4f71b4744b1fec79b1e1e260784d68ef8ec4f6 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Fri, 18 Nov 2016 13:05:31 -0800 Subject: [PATCH 07/10] Add var walker helper --- ast/varset.go | 8 ++++++++ ast/visit.go | 12 ++++++++++++ ast/visit_test.go | 13 +++++++++++++ 3 files changed, 33 insertions(+) diff --git a/ast/varset.go b/ast/varset.go index 176c9ea2e2..c5d4ad0cef 100644 --- a/ast/varset.go +++ b/ast/varset.go @@ -52,6 +52,14 @@ func (s VarSet) Diff(vs VarSet) VarSet { return r } +// Equal returns true if s contains exactly the same elements as vs. +func (s VarSet) Equal(vs VarSet) bool { + if len(s.Diff(vs)) > 0 { + return false + } + return len(vs.Diff(s)) == 0 +} + // Intersect returns a VarSet containing variables in s that are in vs. func (s VarSet) Intersect(vs VarSet) VarSet { r := VarSet{} diff --git a/ast/visit.go b/ast/visit.go index 2a4f9ab545..102013abf3 100644 --- a/ast/visit.go +++ b/ast/visit.go @@ -107,6 +107,18 @@ func WalkRefs(x interface{}, f func(Ref) bool) { Walk(vis, x) } +// WalkVars calls the function f on all vars under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkVars(x interface{}, f func(Var) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if v, ok := x.(Var); ok { + return f(v) + } + return false + }} + Walk(vis, x) +} + // WalkBodies calls the function f on all bodies under x. If the function f // returns true, AST nodes under the last node will not be visited. func WalkBodies(x interface{}, f func(Body) bool) { diff --git a/ast/visit_test.go b/ast/visit_test.go index 4ae0d9a1cf..1524999ef7 100644 --- a/ast/visit_test.go +++ b/ast/visit_test.go @@ -92,3 +92,16 @@ func TestVisitor(t *testing.T) { } } + +func TestWalkVars(t *testing.T) { + x := MustParseBody("x = 1, data.abc[2] = y, y[z] = [q | q = 1]") + found := NewVarSet() + WalkVars(x, func(v Var) bool { + found.Add(v) + return false + }) + expected := NewVarSet(Var("x"), Var("data"), Var("y"), Var("z"), Var("q"), Var("eq")) + if !expected.Equal(found) { + t.Fatalf("Expected %v but got: %v", expected, found) + } +} From a2c7dfb297feaae6194e3136dac9bf07e8f70384 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Fri, 18 Nov 2016 14:29:36 -0800 Subject: [PATCH 08/10] Update REST API to support topdown.QueryResultSet --- server/server.go | 67 +++++++++++++++++++++++++++++++++++++------ server/server_test.go | 11 +++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/server/server.go b/server/server.go index 92bc90c181..c7d19729af 100644 --- a/server/server.go +++ b/server/server.go @@ -89,8 +89,32 @@ func (p *policyV1) Equal(other *policyV1) bool { return p.ID == other.ID && p.Module.Equal(other.Module) } -// resultSetV1 models the result of an ad-hoc query. -type resultSetV1 []map[string]interface{} +// adhocQueryResultSet models the result of a Query API query. +type adhocQueryResultSetV1 []map[string]interface{} + +// queryResultSetV1 models the result of a Data API query when the query would +// return multiple values for the document. +type queryResultSetV1 []*queryResultV1 + +func newQueryResultSetV1(qrs topdown.QueryResultSet) queryResultSetV1 { + result := make(queryResultSetV1, len(qrs)) + for i := range qrs { + result[i] = &queryResultV1{qrs[i].Result, qrs[i].Globals} + } + return result +} + +// queryResultV1 models a single result of a Data API query that would return +// multiple values for the document. The globals can be used to differentiate +// between results. +type queryResultV1 struct { + result interface{} + globals map[string]interface{} +} + +func (qr *queryResultV1) MarshalJSON() ([]byte, error) { + return json.Marshal([]interface{}{qr.result, qr.globals}) +} // explainModeV1 defines supported values for the "explain" query parameter. type explainModeV1 string @@ -310,7 +334,7 @@ func (s *Server) execQuery(compiler *ast.Compiler, txn storage.Transaction, quer ctx.Tracer = buf } - resultSet := resultSetV1{} + resultSet := adhocQueryResultSetV1{} err := topdown.Eval(ctx, func(ctx *topdown.Context) error { result := map[string]interface{}{} @@ -404,12 +428,18 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { path := stringPathToDataRef(vars["path"]) pretty := getPretty(r.URL.Query()["pretty"]) explainMode := getExplain(r.URL.Query()["explain"]) - globals, err := parseGlobals(r.URL.Query()["global"]) + globals, nonGround, err := parseGlobals(r.URL.Query()["global"]) + if err != nil { handleError(w, 400, err) return } + if nonGround && explainMode != explainOffV1 { + handleError(w, 400, fmt.Errorf("explanations with non-ground global values not supported")) + return + } + // Prepare for query. txn, err := s.store.NewTransaction() if err != nil { @@ -446,7 +476,11 @@ func (s *Server) v1DataGet(w http.ResponseWriter, r *http.Request) { return } - // TODO(tsandall): update to handle non-ground global case + if nonGround { + handleResponseJSON(w, 200, newQueryResultSetV1(qrs), pretty) + return + } + result := qrs[0].Result switch explainMode { @@ -943,24 +977,39 @@ func getExplain(p []string) explainModeV1 { return explainOffV1 } -func parseGlobals(s []string) (*ast.ValueMap, error) { +func parseGlobals(s []string) (*ast.ValueMap, bool, error) { pairs := make([][2]*ast.Term, len(s)) + nonGround := false for i := range s { vs := strings.SplitN(s[i], ":", 2) k, err := ast.ParseTerm(vs[0]) if err != nil { - return nil, err + return nil, false, err } v, err := ast.ParseTerm(vs[1]) if err != nil { - return nil, err + return nil, false, err } pairs[i] = [...]*ast.Term{k, v} + if !nonGround { + ast.WalkVars(v, func(x ast.Var) bool { + if x.Equal(ast.DefaultRootDocument.Value) { + return false + } + nonGround = true + return true + }) + } + } + + globals, err := topdown.MakeGlobals(pairs) + if err != nil { + return nil, false, err } - return topdown.MakeGlobals(pairs) + return globals, nonGround, nil } func renderBanner(w http.ResponseWriter) { diff --git a/server/server_test.go b/server/server_test.go index 19e421b135..31821b2d6f 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -57,6 +57,9 @@ func TestDataV1(t *testing.T) { g :- req1.a[0] = 1, reqx.b[i] = 1 h :- attr1[i] > 1 + gt1 :- req1 > 1 + arr = [1,2,3,4] + undef :- false ` @@ -176,6 +179,10 @@ func TestDataV1(t *testing.T) { tr{"PUT", "/policies/test", testMod1, 200, ""}, tr{"GET", "/data/testmod/h?global=req3.attr1%3A%5B4%2C3%2C2%2C1%5D", "", 200, `true`}, }}, + {"get with global (non-ground ref)", []tr{ + tr{"PUT", "/policies/test", testMod1, 200, ""}, + tr{"GET", "/data/testmod/gt1?global=req1:data.testmod.arr[i]", "", 200, `[[true, {"i": 1}], [true, {"i": 2}], [true, {"i": 3}]]`}, + }}, {"get undefined", []tr{ tr{"PUT", "/policies/test", testMod1, 200, ""}, tr{"GET", "/data/testmod/undef", "", 404, ""}, @@ -586,13 +593,13 @@ func TestQueryV1(t *testing.T) { return } - var expected resultSetV1 + var expected adhocQueryResultSetV1 err := json.Unmarshal([]byte(`[{"a":[1,2,3],"i":0,"x":1},{"a":[1,2,3],"i":1,"x":2},{"a":[1,2,3],"i":2,"x":3}]`), &expected) if err != nil { panic(err) } - var result resultSetV1 + var result adhocQueryResultSetV1 err = json.Unmarshal(f.recorder.Body.Bytes(), &result) if err != nil { t.Errorf("Unexpected error while unmarshalling result: %v", err) From adf4ab8435d30a332a2b254988d18503b210bfa5 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Sun, 20 Nov 2016 19:32:45 -0800 Subject: [PATCH 09/10] Check global parameter parse errors --- server/server.go | 3 +++ server/server_test.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/server/server.go b/server/server.go index c7d19729af..a821698313 100644 --- a/server/server.go +++ b/server/server.go @@ -984,6 +984,9 @@ func parseGlobals(s []string) (*ast.ValueMap, bool, error) { for i := range s { vs := strings.SplitN(s[i], ":", 2) + if len(vs) != 2 { + return nil, false, fmt.Errorf("global format: : where is either var or ref") + } k, err := ast.ParseTerm(vs[0]) if err != nil { return nil, false, err diff --git a/server/server_test.go b/server/server_test.go index 31821b2d6f..f179a5698e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -183,6 +183,12 @@ func TestDataV1(t *testing.T) { tr{"PUT", "/policies/test", testMod1, 200, ""}, tr{"GET", "/data/testmod/gt1?global=req1:data.testmod.arr[i]", "", 200, `[[true, {"i": 1}], [true, {"i": 2}], [true, {"i": 3}]]`}, }}, + {"get with global (bad format)", []tr{ + tr{"GET", "/data/deadbeef?global=[1,2,3]", "", 400, `{ + "Code": 400, + "Message": "global format: : where is either var or ref" + }`}, + }}, {"get undefined", []tr{ tr{"PUT", "/policies/test", testMod1, 200, ""}, tr{"GET", "/data/testmod/undef", "", 404, ""}, From 4924c84ad5589f3aa299e7f4bfe8eab107d205b0 Mon Sep 17 00:00:00 2001 From: Torin Sandall Date: Sun, 20 Nov 2016 20:16:02 -0800 Subject: [PATCH 10/10] Update REST API docs with result set example --- site/_scripts/rest-examples/containers.json | 330 ++++++++++++++++ site/_scripts/rest-examples/example4.rego | 9 + site/_scripts/rest-examples/gen-examples.sh | 4 + site/documentation/references/rest/index.md | 410 +++++++++++++++++++- 4 files changed, 747 insertions(+), 6 deletions(-) create mode 100644 site/_scripts/rest-examples/containers.json create mode 100644 site/_scripts/rest-examples/example4.rego diff --git a/site/_scripts/rest-examples/containers.json b/site/_scripts/rest-examples/containers.json new file mode 100644 index 0000000000..1872f7bffd --- /dev/null +++ b/site/_scripts/rest-examples/containers.json @@ -0,0 +1,330 @@ +[ + { + "Id": "a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052", + "Created": "2016-11-21T03:13:14.288557666Z", + "Path": "sh", + "Args": [], + "State": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 12127, + "ExitCode": 0, + "Error": "", + "StartedAt": "2016-11-21T03:13:14.915355869Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "Image": "sha256:baa5d63471ead618ff91ddfacf1e2c81bf0612bfeb1daf00eb0843a41fbfade3", + "ResolvConfPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/resolv.conf", + "HostnamePath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/hostname", + "HostsPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/hosts", + "LogPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052-json.log", + "Name": "/suspicious_brahmagupta", + "RestartCount": 0, + "Driver": "aufs", + "MountLabel": "", + "ProcessLabel": "", + "AppArmorProfile": "", + "ExecIDs": null, + "HostConfig": { + "Binds": null, + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "default", + "PortBindings": {}, + "RestartPolicy": { + "Name": "no", + "MaximumRetryCount": 0 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "CapAdd": null, + "CapDrop": null, + "Dns": [], + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": false, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": null, + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "ConsoleSize": [ + 0, + 0 + ], + "Isolation": "", + "CpuShares": 0, + "Memory": 0, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": null, + "BlkioDeviceReadBps": null, + "BlkioDeviceWriteBps": null, + "BlkioDeviceReadIOps": null, + "BlkioDeviceWriteIOps": null, + "CpuPeriod": 0, + "CpuQuota": 0, + "CpusetCpus": "", + "CpusetMems": "", + "Devices": [], + "DiskQuota": 0, + "KernelMemory": 0, + "MemoryReservation": 0, + "MemorySwap": 0, + "MemorySwappiness": -1, + "OomKillDisable": false, + "PidsLimit": 0, + "Ulimits": null, + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0 + }, + "GraphDriver": { + "Name": "aufs", + "Data": null + }, + "Mounts": [], + "Config": { + "Hostname": "a4288db7773e", + "Domainname": "", + "User": "", + "AttachStdin": true, + "AttachStdout": true, + "AttachStderr": true, + "Tty": true, + "OpenStdin": true, + "StdinOnce": true, + "Env": [ + "no_proxy=*.local, 169.254/16", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "sh" + ], + "Image": "alpine:latest", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": {} + }, + "NetworkSettings": { + "Bridge": "", + "SandboxID": "2dadac1a8b18b7ae5658c4215637d998572c95bc0673bac2aceefbdd830d8860", + "HairpinMode": false, + "LinkLocalIPv6Address": "", + "LinkLocalIPv6PrefixLen": 0, + "Ports": {}, + "SandboxKey": "/var/run/docker/netns/2dadac1a8b18", + "SecondaryIPAddresses": null, + "SecondaryIPv6Addresses": null, + "EndpointID": "fb1af875e2f0e7643224b6505a2c713748175689b8e8edb7cc1496efa8cdcafd", + "Gateway": "172.17.0.1", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "IPAddress": "172.17.0.3", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "MacAddress": "02:42:ac:11:00:03", + "Networks": { + "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "ad1022afa0af59671f2b701ff8cbd4607de24740b59484acd4a740fac4ad26f9", + "EndpointID": "fb1af875e2f0e7643224b6505a2c713748175689b8e8edb7cc1496efa8cdcafd", + "Gateway": "172.17.0.1", + "IPAddress": "172.17.0.3", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "02:42:ac:11:00:03" + } + } + } + }, + { + "Id": "6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218", + "Created": "2016-11-21T03:13:05.080079329Z", + "Path": "sh", + "Args": [], + "State": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 12079, + "ExitCode": 0, + "Error": "", + "StartedAt": "2016-11-21T03:13:05.718411107Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "Image": "sha256:baa5d63471ead618ff91ddfacf1e2c81bf0612bfeb1daf00eb0843a41fbfade3", + "ResolvConfPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/resolv.conf", + "HostnamePath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/hostname", + "HostsPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/hosts", + "LogPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218-json.log", + "Name": "/fervent_almeida", + "RestartCount": 0, + "Driver": "aufs", + "MountLabel": "", + "ProcessLabel": "", + "AppArmorProfile": "", + "ExecIDs": null, + "HostConfig": { + "Binds": null, + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "default", + "PortBindings": {}, + "RestartPolicy": { + "Name": "no", + "MaximumRetryCount": 0 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "CapAdd": null, + "CapDrop": null, + "Dns": [], + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": false, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": [ + "seccomp:unconfined" + ], + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "ConsoleSize": [ + 0, + 0 + ], + "Isolation": "", + "CpuShares": 0, + "Memory": 0, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": null, + "BlkioDeviceReadBps": null, + "BlkioDeviceWriteBps": null, + "BlkioDeviceReadIOps": null, + "BlkioDeviceWriteIOps": null, + "CpuPeriod": 0, + "CpuQuota": 0, + "CpusetCpus": "", + "CpusetMems": "", + "Devices": [], + "DiskQuota": 0, + "KernelMemory": 0, + "MemoryReservation": 0, + "MemorySwap": 0, + "MemorySwappiness": -1, + "OomKillDisable": false, + "PidsLimit": 0, + "Ulimits": null, + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0 + }, + "GraphDriver": { + "Name": "aufs", + "Data": null + }, + "Mounts": [], + "Config": { + "Hostname": "6887a5168d0e", + "Domainname": "", + "User": "", + "AttachStdin": true, + "AttachStdout": true, + "AttachStderr": true, + "Tty": true, + "OpenStdin": true, + "StdinOnce": true, + "Env": [ + "no_proxy=*.local, 169.254/16", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "sh" + ], + "Image": "alpine:latest", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": {} + }, + "NetworkSettings": { + "Bridge": "", + "SandboxID": "580a6d3f8bebfb1647c8b34f7766dee1b86443752ef3e255d37517dfb35076ed", + "HairpinMode": false, + "LinkLocalIPv6Address": "", + "LinkLocalIPv6PrefixLen": 0, + "Ports": {}, + "SandboxKey": "/var/run/docker/netns/580a6d3f8beb", + "SecondaryIPAddresses": null, + "SecondaryIPv6Addresses": null, + "EndpointID": "a4340ac5857a79310cc7eaa344d4ff6771fdf6372f72447b91928c038d9ada7d", + "Gateway": "172.17.0.1", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "IPAddress": "172.17.0.2", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "MacAddress": "02:42:ac:11:00:02", + "Networks": { + "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "ad1022afa0af59671f2b701ff8cbd4607de24740b59484acd4a740fac4ad26f9", + "EndpointID": "a4340ac5857a79310cc7eaa344d4ff6771fdf6372f72447b91928c038d9ada7d", + "Gateway": "172.17.0.1", + "IPAddress": "172.17.0.2", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "02:42:ac:11:00:02" + } + } + } + } +] diff --git a/site/_scripts/rest-examples/example4.rego b/site/_scripts/rest-examples/example4.rego new file mode 100644 index 0000000000..bb00136eee --- /dev/null +++ b/site/_scripts/rest-examples/example4.rego @@ -0,0 +1,9 @@ +package opa.examples + +import container + +allow_container :- + not seccomp_unconfined + +seccomp_unconfined :- + container.HostConfig.SecurityOpt[_] = "seccomp:unconfined" diff --git a/site/_scripts/rest-examples/gen-examples.sh b/site/_scripts/rest-examples/gen-examples.sh index 267decc3ad..7bb336c94f 100755 --- a/site/_scripts/rest-examples/gen-examples.sh +++ b/site/_scripts/rest-examples/gen-examples.sh @@ -65,6 +65,8 @@ function delete_a_policy() { function get_a_document() { # Create policy module for this example. curl $BASE_URL/policies/example3 -X PUT -s -v --data-binary @example3.rego >/dev/null 2>&1 + curl $BASE_URL/policies/example4 -X PUT -s -v --data-binary @example4.rego >/dev/null 2>&1 + curl $BASE_URL/data/containers -X PUT -s -v --data-binary @containers.json >/dev/null 2>&1 echo "" echo "### Get a Document" @@ -73,6 +75,8 @@ function get_a_document() { echo "" curl $BASE_URL/data/opa/examples/allow_request?pretty=true -s -v -G --data-urlencode 'global=example.flag:false' echo "" + curl $BASE_URL/data/opa/examples/allow_container?pretty=true -s -v -G --data-urlencode 'global=container:data.containers[container_index]' + echo "" # Delete policy module created above. curl $BASE_URL/policies/example3 -X DELETE -s >/dev/null 2>&1 diff --git a/site/documentation/references/rest/index.md b/site/documentation/references/rest/index.md index 78967ac704..61c64e5ebb 100644 --- a/site/documentation/references/rest/index.md +++ b/site/documentation/references/rest/index.md @@ -1120,7 +1120,7 @@ Content-Type: application/json #### Query Parameters -- **global** - Provide an input document to the query. Format is `:` where `` is the import path of the input document and `` is the JSON serialized input document. The parameter may be specified multiple times but each instance should specify a unique ``. +- **global** - Provide an input document to the query. Format is `:` where `` is the import path of the input document. The parameter may be specified multiple times but each instance should specify a unique ``. The `` may be a reference to a document in OPA. If `` contains variables the response will contain a set of results instead of a single document. - **pretty** - If parameter is `true`, response will formatted for humans. - **explain** - Return query explanation instead of normal result. Values: **full**, **truth**. See [Explanations](#explanations) for how to interpret results. @@ -1138,7 +1138,20 @@ The server returns 404 in two cases: - The path refers to a non-existent base document. - The path refers to a Virtual Document that is undefined in the context of the query. -#### Example Module +#### Example Request With Global Parameter + +```http +GET /v1/data/opa/examples/allow_request?global=example.flag:false HTTP/1.1 +``` + +#### Example Response For Non-Existent Or Undefined Document + +```http +HTTP/1.1 404 Not Found +``` + +The example above assumes the following policy: + ```ruby package opa.examples @@ -1148,16 +1161,401 @@ import example.flag allow_request :- flag = true ``` -#### Example Request With Global Parameter +#### Example Request For Result Set ```http -GET /v1/data/opa/examples/allow_request?global=example.flag:false HTTP/1.1 +GET /v1/data/opa/examples/allow_container?global=data.containers[container_index] HTTP/1.1 ``` -#### Example Response For Non-Existent Or Undefined Document +#### Example Response For Result Set ```http -HTTP/1.1 404 Not Found +HTTP/1.1 200 OK +Content-Type: application/json + +[ + [ + true, + { + "container_index": 0 + } + ] +] +``` + +Result sets have the following schema: + +```json +{ + "type": "array", + "title": "Result Set", + "description": "Set of results returned for a Data API query.", + "items": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": [ + { + "title": "Value", + "description": "The value of the document referred to by the Data API path." + }, + { + "type": "object", + "title": "Bindings", + "description": "The bindings of variables found in the Data API query input documents." + } + ] + } +} +``` + +The example above assumes the following data and policy: + + +```json +{ + "containers": [ + { + "Id": "a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052", + "Created": "2016-11-21T03:13:14.288557666Z", + "Path": "sh", + "Args": [], + "State": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 12127, + "ExitCode": 0, + "Error": "", + "StartedAt": "2016-11-21T03:13:14.915355869Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "Image": "sha256:baa5d63471ead618ff91ddfacf1e2c81bf0612bfeb1daf00eb0843a41fbfade3", + "ResolvConfPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/resolv.conf", + "HostnamePath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/hostname", + "HostsPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/hosts", + "LogPath": "/var/lib/docker/containers/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052/a4288db7773ebb4f1b4d502712b87b241e3c28184cda6a1ad58f91ac6d89f052-json.log", + "Name": "/suspicious_brahmagupta", + "RestartCount": 0, + "Driver": "aufs", + "MountLabel": "", + "ProcessLabel": "", + "AppArmorProfile": "", + "ExecIDs": null, + "HostConfig": { + "Binds": null, + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "default", + "PortBindings": {}, + "RestartPolicy": { + "Name": "no", + "MaximumRetryCount": 0 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "CapAdd": null, + "CapDrop": null, + "Dns": [], + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": false, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": null, + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "ConsoleSize": [ + 0, + 0 + ], + "Isolation": "", + "CpuShares": 0, + "Memory": 0, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": null, + "BlkioDeviceReadBps": null, + "BlkioDeviceWriteBps": null, + "BlkioDeviceReadIOps": null, + "BlkioDeviceWriteIOps": null, + "CpuPeriod": 0, + "CpuQuota": 0, + "CpusetCpus": "", + "CpusetMems": "", + "Devices": [], + "DiskQuota": 0, + "KernelMemory": 0, + "MemoryReservation": 0, + "MemorySwap": 0, + "MemorySwappiness": -1, + "OomKillDisable": false, + "PidsLimit": 0, + "Ulimits": null, + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0 + }, + "GraphDriver": { + "Name": "aufs", + "Data": null + }, + "Mounts": [], + "Config": { + "Hostname": "a4288db7773e", + "Domainname": "", + "User": "", + "AttachStdin": true, + "AttachStdout": true, + "AttachStderr": true, + "Tty": true, + "OpenStdin": true, + "StdinOnce": true, + "Env": [ + "no_proxy=*.local, 169.254/16", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "sh" + ], + "Image": "alpine:latest", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": {} + }, + "NetworkSettings": { + "Bridge": "", + "SandboxID": "2dadac1a8b18b7ae5658c4215637d998572c95bc0673bac2aceefbdd830d8860", + "HairpinMode": false, + "LinkLocalIPv6Address": "", + "LinkLocalIPv6PrefixLen": 0, + "Ports": {}, + "SandboxKey": "/var/run/docker/netns/2dadac1a8b18", + "SecondaryIPAddresses": null, + "SecondaryIPv6Addresses": null, + "EndpointID": "fb1af875e2f0e7643224b6505a2c713748175689b8e8edb7cc1496efa8cdcafd", + "Gateway": "172.17.0.1", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "IPAddress": "172.17.0.3", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "MacAddress": "02:42:ac:11:00:03", + "Networks": { + "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "ad1022afa0af59671f2b701ff8cbd4607de24740b59484acd4a740fac4ad26f9", + "EndpointID": "fb1af875e2f0e7643224b6505a2c713748175689b8e8edb7cc1496efa8cdcafd", + "Gateway": "172.17.0.1", + "IPAddress": "172.17.0.3", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "02:42:ac:11:00:03" + } + } + } + }, + { + "Id": "6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218", + "Created": "2016-11-21T03:13:05.080079329Z", + "Path": "sh", + "Args": [], + "State": { + "Status": "running", + "Running": true, + "Paused": false, + "Restarting": false, + "OOMKilled": false, + "Dead": false, + "Pid": 12079, + "ExitCode": 0, + "Error": "", + "StartedAt": "2016-11-21T03:13:05.718411107Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "Image": "sha256:baa5d63471ead618ff91ddfacf1e2c81bf0612bfeb1daf00eb0843a41fbfade3", + "ResolvConfPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/resolv.conf", + "HostnamePath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/hostname", + "HostsPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/hosts", + "LogPath": "/var/lib/docker/containers/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218/6887a5168d0e2324b8c9544c4b30321bce4952f12698491f268c17bbe77e5218-json.log", + "Name": "/fervent_almeida", + "RestartCount": 0, + "Driver": "aufs", + "MountLabel": "", + "ProcessLabel": "", + "AppArmorProfile": "", + "ExecIDs": null, + "HostConfig": { + "Binds": null, + "ContainerIDFile": "", + "LogConfig": { + "Type": "json-file", + "Config": {} + }, + "NetworkMode": "default", + "PortBindings": {}, + "RestartPolicy": { + "Name": "no", + "MaximumRetryCount": 0 + }, + "AutoRemove": false, + "VolumeDriver": "", + "VolumesFrom": null, + "CapAdd": null, + "CapDrop": null, + "Dns": [], + "DnsOptions": [], + "DnsSearch": [], + "ExtraHosts": null, + "GroupAdd": null, + "IpcMode": "", + "Cgroup": "", + "Links": null, + "OomScoreAdj": 0, + "PidMode": "", + "Privileged": false, + "PublishAllPorts": false, + "ReadonlyRootfs": false, + "SecurityOpt": [ + "seccomp:unconfined" + ], + "UTSMode": "", + "UsernsMode": "", + "ShmSize": 67108864, + "Runtime": "runc", + "ConsoleSize": [ + 0, + 0 + ], + "Isolation": "", + "CpuShares": 0, + "Memory": 0, + "CgroupParent": "", + "BlkioWeight": 0, + "BlkioWeightDevice": null, + "BlkioDeviceReadBps": null, + "BlkioDeviceWriteBps": null, + "BlkioDeviceReadIOps": null, + "BlkioDeviceWriteIOps": null, + "CpuPeriod": 0, + "CpuQuota": 0, + "CpusetCpus": "", + "CpusetMems": "", + "Devices": [], + "DiskQuota": 0, + "KernelMemory": 0, + "MemoryReservation": 0, + "MemorySwap": 0, + "MemorySwappiness": -1, + "OomKillDisable": false, + "PidsLimit": 0, + "Ulimits": null, + "CpuCount": 0, + "CpuPercent": 0, + "IOMaximumIOps": 0, + "IOMaximumBandwidth": 0 + }, + "GraphDriver": { + "Name": "aufs", + "Data": null + }, + "Mounts": [], + "Config": { + "Hostname": "6887a5168d0e", + "Domainname": "", + "User": "", + "AttachStdin": true, + "AttachStdout": true, + "AttachStderr": true, + "Tty": true, + "OpenStdin": true, + "StdinOnce": true, + "Env": [ + "no_proxy=*.local, 169.254/16", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "Cmd": [ + "sh" + ], + "Image": "alpine:latest", + "Volumes": null, + "WorkingDir": "", + "Entrypoint": null, + "OnBuild": null, + "Labels": {} + }, + "NetworkSettings": { + "Bridge": "", + "SandboxID": "580a6d3f8bebfb1647c8b34f7766dee1b86443752ef3e255d37517dfb35076ed", + "HairpinMode": false, + "LinkLocalIPv6Address": "", + "LinkLocalIPv6PrefixLen": 0, + "Ports": {}, + "SandboxKey": "/var/run/docker/netns/580a6d3f8beb", + "SecondaryIPAddresses": null, + "SecondaryIPv6Addresses": null, + "EndpointID": "a4340ac5857a79310cc7eaa344d4ff6771fdf6372f72447b91928c038d9ada7d", + "Gateway": "172.17.0.1", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "IPAddress": "172.17.0.2", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "MacAddress": "02:42:ac:11:00:02", + "Networks": { + "bridge": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "ad1022afa0af59671f2b701ff8cbd4607de24740b59484acd4a740fac4ad26f9", + "EndpointID": "a4340ac5857a79310cc7eaa344d4ff6771fdf6372f72447b91928c038d9ada7d", + "Gateway": "172.17.0.1", + "IPAddress": "172.17.0.2", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "02:42:ac:11:00:02" + } + } + } + }] +} +``` + +```ruby +package opa.examples + +import container + +allow_container :- + not seccomp_unconfined + +seccomp_unconfined :- + container.HostConfig.SecurityOpt[_] = "seccomp:unconfined" ``` ### Create or Overwrite a Document