diff --git a/docs/faq.md b/docs/faq.md index e9e8925df9..d8bf274213 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -242,7 +242,7 @@ Similar logic applies for situations when you only specify the `mode` parameter. This all turns out to be more safe and "correct", in that it would error and prevent masking an error for a situation when you expected a file to already be at that location. It also turns out to simplify the internals significantly, and -remove an ambiguous scenario with the reversable file resource. +remove an ambiguous scenario with the reversible file resource. ### Why do function names inside of templates include underscores? diff --git a/docs/language-guide.md b/docs/language-guide.md index a94d6983d3..d914cd6a49 100644 --- a/docs/language-guide.md +++ b/docs/language-guide.md @@ -563,11 +563,11 @@ would like to propose a more logical or performant variant. #### Function graph generation -At this point we have a fully type AST. The AST must now be transformed into a +At this point we have a fully typed AST. The AST must now be transformed into a directed, acyclic graph (DAG) data structure that represents the flow of data as necessary for everything to be reactive. Note that this graph is *different* from the resource graph which is produced and sent to the engine. It is just a -coincidence that both happen to be DAG's. (You don't freak out when you see a +coincidence that both happen to be DAG's. (You aren't surprised when you see a list data structure show up in more than one place, do you?) To produce this graph, each node has a `Graph` method which it can call. This @@ -575,9 +575,8 @@ starts at the top most node, and is called down through the AST. The edges in the graphs must represent the individual expression values which are passed from node to node. The names of the edges must match the function type argument names which are used in the definition of the corresponding function. These -corresponding functions must exist for each expression node and are produced by -calling that expression's `Func` method. These are usually called by the -function engine during function creation and validation. +corresponding functions must exist for each expression node and are produced as +the vertices of this returned graph. This is built for the function engine. #### Function engine creation and validation diff --git a/examples/lang/map-iterator0.mcl b/examples/lang/map-iterator0.mcl new file mode 100644 index 0000000000..316555ca74 --- /dev/null +++ b/examples/lang/map-iterator0.mcl @@ -0,0 +1,13 @@ +import "iter" + +$fn = func($x) { # notable because concrete type is fn(t1) t2, where t1 != t2 + len($x) +} + +$in1 = ["a", "bb", "ccc", "dddd", "eeeee",] + +$out1 = iter.map($in1, $fn) + +$t1 = template("out1: {{ . }}", $out1) + +test $t1 {} diff --git a/examples/lang/map-iterator1.mcl b/examples/lang/map-iterator1.mcl new file mode 100644 index 0000000000..861652297a --- /dev/null +++ b/examples/lang/map-iterator1.mcl @@ -0,0 +1,28 @@ +import "datetime" +import "iter" +import "math" + +$now = datetime.now() + +# alternate every four seconds +$mod0 = math.mod($now, 8) == 0 +$mod1 = math.mod($now, 8) == 1 +$mod2 = math.mod($now, 8) == 2 +$mod3 = math.mod($now, 8) == 3 +$mod = $mod0 || $mod1 || $mod2 || $mod3 + +$fn = func($x) { # notable because concrete type is fn(t1) t2, where t1 != t2 + len($x) +} + +$in1 = if $mod { + ["a", "bb", "ccc", "dddd", "eeeee",] +} else { + ["ffffff", "ggggggg", "hhhhhhhh", "iiiiiiiii", "jjjjjjjjjj",] +} + +$out1 = iter.map($in1, $fn) + +$t1 = template("out1: {{ . }}", $out1) + +test $t1 {} diff --git a/gapi/gapi.go b/gapi/gapi.go index fbadc9c46a..c10d6ff76a 100644 --- a/gapi/gapi.go +++ b/gapi/gapi.go @@ -119,11 +119,13 @@ type GAPI interface { // Next returns a stream of switch events. The engine will run Graph() // to build a new graph after every Next event. + // TODO: add context for shutting down to the input and change Close to Cleanup Next() chan Next // Close shuts down the GAPI. It asks the GAPI to close, and must cause // Next() to unblock even if is currently blocked and waiting to send a // new event. + // TODO: change Close to Cleanup Close() error } diff --git a/lang/ast/scope_test.go b/lang/ast/scope_test.go deleted file mode 100644 index b2c01903b1..0000000000 --- a/lang/ast/scope_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2023+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//go:build !root - -package ast - -import ( - "fmt" - "reflect" - "testing" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/util" -) - -func TestScopeIndexesPush0(t *testing.T) { - type test struct { // an individual test - name string - indexes map[int][]interfaces.Expr - pushed []interfaces.Expr - expected map[int][]interfaces.Expr - } - testCases := []test{} - - //{ - // testCases = append(testCases, test{ - // name: "empty", - // pushed: nil, // TODO: undefined, but should we do it? - // expected: map[int][]interfaces.Expr{ - // 0: {}, // empty list ? - // }, - // }) - //} - { - testCases = append(testCases, test{ - name: "empty list", - pushed: []interfaces.Expr{}, // empty list - expected: map[int][]interfaces.Expr{ - 0: {}, // empty list - }, - }) - } - { - b1 := &ExprBool{} - b2 := &ExprBool{} - b3 := &ExprBool{} - b4 := &ExprBool{} - b5 := &ExprBool{} - b6 := &ExprBool{} - b7 := &ExprBool{} - b8 := &ExprBool{} - testCases = append(testCases, test{ - name: "simple push", - indexes: map[int][]interfaces.Expr{ - 0: { - b1, b2, b3, - }, - 1: { - b4, - }, - 2: { - b5, b6, - }, - }, - pushed: []interfaces.Expr{ - b7, b8, - }, - expected: map[int][]interfaces.Expr{ - 0: { - b7, b8, - }, - 1: { - b1, b2, b3, - }, - 2: { - b4, - }, - 3: { - b5, b6, - }, - }, - }) - } - { - b1 := &ExprBool{} - b2 := &ExprBool{} - b3 := &ExprBool{} - b4 := &ExprBool{} - b5 := &ExprBool{} - b6 := &ExprBool{} - b7 := &ExprBool{} - b8 := &ExprBool{} - testCases = append(testCases, test{ - name: "push with gaps", - indexes: map[int][]interfaces.Expr{ - 0: { - b1, b2, b3, - }, - // there is a gap here - 2: { - b4, - }, - 3: { - b5, b6, - }, - }, - pushed: []interfaces.Expr{ - b7, b8, - }, - expected: map[int][]interfaces.Expr{ - 0: { - b7, b8, - }, - // the gap remains - 1: { - b1, b2, b3, - }, - 3: { - b4, - }, - 4: { - b5, b6, - }, - }, - }) - } - names := []string{} - for index, tc := range testCases { // run all the tests - if tc.name == "" { - t.Errorf("test #%d: not named", index) - continue - } - if util.StrInList(tc.name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) - continue - } - names = append(names, tc.name) - - //if index != 3 { // hack to run a subset (useful for debugging) - //if (index != 20 && index != 21) { - //if tc.name != "nil" { - // continue - //} - - t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { - name, indexes, pushed, expected := tc.name, tc.indexes, tc.pushed, tc.expected - - t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) - - scope := &interfaces.Scope{ - Indexes: indexes, - } - scope.PushIndexes(pushed) - out := scope.Indexes - - if !reflect.DeepEqual(out, expected) { - t.Errorf("test #%d: indexes did not match expected", index) - t.Logf("test #%d: actual: \n\n%+v\n", index, out) - t.Logf("test #%d: expected: \n\n%+v", index, expected) - return - } - }) - } -} diff --git a/lang/ast/scopegraph.go b/lang/ast/scopegraph.go new file mode 100644 index 0000000000..374633f883 --- /dev/null +++ b/lang/ast/scopegraph.go @@ -0,0 +1,201 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package ast + +import ( + "fmt" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/pgraph" +) + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtBind) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtRes) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + name, ok := obj.Name.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + name.ScopeGraph(g) + g.AddEdge(obj, obj.Name, &pgraph.SimpleEdge{Name: "name"}) + for _, resContents := range obj.Contents { + switch r := resContents.(type) { + case *StmtResField: + rValue, ok := r.Value.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + rValue.ScopeGraph(g) + g.AddEdge(obj, r.Value, &pgraph.SimpleEdge{Name: r.Field}) + } + } +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtEdge) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtIf) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtProg) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + for _, stmt := range obj.Body { + stmt, ok := stmt.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + stmt.ScopeGraph(g) + g.AddEdge(obj, stmt, &pgraph.SimpleEdge{Name: ""}) + } +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtFunc) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtClass) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtInclude) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtImport) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *StmtComment) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprBool) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprStr) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprInt) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprFloat) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprList) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprMap) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprStruct) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprFunc) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + if obj.Body != nil { + body, ok := obj.Body.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + body.ScopeGraph(g) + g.AddEdge(obj, obj.Body, &pgraph.SimpleEdge{Name: "body"}) + } +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprCall) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + expr, ok := obj.expr.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + expr.ScopeGraph(g) + g.AddEdge(obj, obj.expr, &pgraph.SimpleEdge{Name: "expr"}) + + for i, arg := range obj.Args { + arg, ok := arg.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + arg.ScopeGraph(g) + g.AddEdge(obj, arg, &pgraph.SimpleEdge{Name: fmt.Sprintf("arg%d", i)}) + } +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprVar) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + target := obj.scope.Variables[obj.Name] + newTarget, ok := target.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + newTarget.ScopeGraph(g) + g.AddEdge(obj, target, &pgraph.SimpleEdge{Name: "target"}) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprParam) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprPoly) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) + definition, ok := obj.Definition.(interfaces.ScopeGrapher) + if !ok { + panic("can't graph scope") // programming error + } + definition.ScopeGraph(g) + g.AddEdge(obj, obj.Definition, &pgraph.SimpleEdge{Name: "def"}) +} + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprIf) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} diff --git a/lang/ast/structs.go b/lang/ast/structs.go index 4fcf123af5..26dfb9b46d 100644 --- a/lang/ast/structs.go +++ b/lang/ast/structs.go @@ -35,6 +35,7 @@ import ( "github.com/purpleidea/mgmt/lang/inputs" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/lang/types/full" langutil "github.com/purpleidea/mgmt/lang/util" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util" @@ -90,6 +91,12 @@ const ( // varOrderingPrefix is a magic prefix used for the Ordering graph. varOrderingPrefix = "var:" + // paramOrderingPrefix is a magic prefix used for the Ordering graph. + paramOrderingPrefix = "param:" + + // recurOrderingPrefix is a magic prefix used for the Ordering graph. + recurOrderingPrefix = "recur:" + // funcOrderingPrefix is a magic prefix used for the Ordering graph. funcOrderingPrefix = "func:" @@ -103,11 +110,20 @@ const ( // ErrNoStoredScope is an error that tells us we can't get a scope here. ErrNoStoredScope = interfaces.Error("scope is not stored in this node") + + // ErrFuncPointerNil is an error that explains the function pointer for + // table lookup is missing. If this happens, it's most likely a + // programming error. + ErrFuncPointerNil = interfaces.Error("missing func pointer for table") + + // ErrTableNoValue is an error that explains the table is missing a + // value. If this happens, it's most likely a programming error. + ErrTableNoValue = interfaces.Error("missing value in table") ) var ( // orderingGraphSingleton is used for debugging the ordering graph. - orderingGraphSingleton = true + orderingGraphSingleton = false ) // StmtBind is a representation of an assignment, which binds a variable to an @@ -218,11 +234,11 @@ func (obj *StmtBind) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap return graph, cons, nil } -// SetScope sets the scope of the child expression bound to it. It seems this is -// necessary in order to reach this, in particular in situations when a bound -// expression points to a previously bound expression. +// SetScope sets the scope of the child expression bound to it. If a variable +// uses the value which this StmtBind binds, they will make a copy and call +// SetScope on the copy. func (obj *StmtBind) SetScope(scope *interfaces.Scope) error { - return obj.Value.SetScope(scope) + return nil } // Unify returns the list of invariants that this node produces. It recursively @@ -255,13 +271,13 @@ func (obj *StmtBind) Graph() (*pgraph.Graph, error) { // It seems that adding this to the graph will end up including an // expression in the case of an ExprFunc lambda, since we copy it and // build a new ExprFunc when it's used by ExprCall. - //return obj.Value.Graph() // nope! - return pgraph.NewGraph("stmtbind") // empty graph! + //return obj.Value.Graph(nil) // nope! + return pgraph.NewGraph("stmtbind") // empty graph } // Output for the bind statement produces no output. Any values of interest come // from the use of the var which this binds the expression to. -func (obj *StmtBind) Output() (*interfaces.Output, error) { +func (obj *StmtBind) Output(map[interfaces.Func]types.Value) (*interfaces.Output, error) { return interfaces.EmptyOutput(), nil } @@ -276,6 +292,7 @@ type StmtRes struct { Kind string // kind of resource, eg: pkg, file, svc, etc... Name interfaces.Expr // unique name for the res of this kind + namePtr interfaces.Func // ptr for table lookup Contents []StmtResContents // list of fields/edges in parsed order } @@ -482,7 +499,7 @@ func (obj *StmtRes) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. func (obj *StmtRes) SetScope(scope *interfaces.Scope) error { - if err := obj.Name.SetScope(scope); err != nil { + if err := obj.Name.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } for _, x := range obj.Contents { @@ -592,11 +609,12 @@ func (obj *StmtRes) Graph() (*pgraph.Graph, error) { return nil, err } - g, err := obj.Name.Graph() + g, f, err := obj.Name.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.namePtr = f for _, x := range obj.Contents { g, err := x.Graph() @@ -614,10 +632,13 @@ func (obj *StmtRes) Graph() (*pgraph.Graph, error) { // analogous function for expressions is Value. Those Value functions might get // called by this Output function if they are needed to produce the output. In // the case of this resource statement, this is definitely the case. -func (obj *StmtRes) Output() (*interfaces.Output, error) { - nameValue, err := obj.Name.Value() - if err != nil { - return nil, err +func (obj *StmtRes) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { + if obj.namePtr == nil { + return nil, ErrFuncPointerNil + } + nameValue, exists := table[obj.namePtr] + if !exists { + return nil, ErrTableNoValue } names := []string{} // list of names to build @@ -640,18 +661,18 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) { resources := []engine.Res{} edges := []*interfaces.Edge{} for _, name := range names { - res, err := obj.resource(name) + res, err := obj.resource(table, name) if err != nil { return nil, errwrap.Wrapf(err, "error building resource") } - edgeList, err := obj.edges(name) + edgeList, err := obj.edges(table, name) if err != nil { return nil, errwrap.Wrapf(err, "error building edges") } edges = append(edges, edgeList...) - if err := obj.metaparams(res); err != nil { // set metaparams + if err := obj.metaparams(table, res); err != nil { // set metaparams return nil, errwrap.Wrapf(err, "error building meta params") } resources = append(resources, res) @@ -665,7 +686,7 @@ func (obj *StmtRes) Output() (*interfaces.Output, error) { // resource is a helper function to generate the res that comes from this. // TODO: it could memoize some of the work to avoid re-computation when looped -func (obj *StmtRes) resource(resName string) (engine.Res, error) { +func (obj *StmtRes) resource(table map[interfaces.Func]types.Value, resName string) (engine.Res, error) { res, err := engine.NewNamedResource(obj.Kind, resName) if err != nil { return nil, errwrap.Wrapf(err, "cannot create resource kind `%s` with named `%s`", obj.Kind, resName) @@ -690,9 +711,12 @@ func (obj *StmtRes) resource(resName string) (engine.Res, error) { } if x.Condition != nil { - b, err := x.Condition.Value() - if err != nil { - return nil, err + if x.conditionPtr == nil { + return nil, ErrFuncPointerNil + } + b, exists := table[x.conditionPtr] + if !exists { + return nil, ErrTableNoValue } if !b.Bool() { // if value exists, and is false, skip it @@ -729,9 +753,12 @@ func (obj *StmtRes) resource(resName string) (engine.Res, error) { return nil, errwrap.Wrapf(err, "resource field `%s` of type `%+v`, cannot take type `%+v", x.Field, t, typ) } - fv, err := x.Value.Value() // Value method on Expr - if err != nil { - return nil, err + if x.valuePtr == nil { + return nil, ErrFuncPointerNil + } + fv, exists := table[x.valuePtr] + if !exists { + return nil, ErrTableNoValue } // mutate the struct field f with the mcl data in fv @@ -744,7 +771,7 @@ func (obj *StmtRes) resource(resName string) (engine.Res, error) { } // edges is a helper function to generate the edges that come from the resource. -func (obj *StmtRes) edges(resName string) ([]*interfaces.Edge, error) { +func (obj *StmtRes) edges(table map[interfaces.Func]types.Value, resName string) ([]*interfaces.Edge, error) { edges := []*interfaces.Edge{} // to and from self, map of kind, name, notify @@ -758,9 +785,12 @@ func (obj *StmtRes) edges(resName string) ([]*interfaces.Edge, error) { } if x.Condition != nil { - b, err := x.Condition.Value() - if err != nil { - return nil, err + if x.conditionPtr == nil { + return nil, ErrFuncPointerNil + } + b, exists := table[x.conditionPtr] + if !exists { + return nil, ErrTableNoValue } if !b.Bool() { // if value exists, and is false, skip it @@ -768,9 +798,12 @@ func (obj *StmtRes) edges(resName string) ([]*interfaces.Edge, error) { } } - nameValue, err := x.EdgeHalf.Name.Value() - if err != nil { - return nil, err + if x.EdgeHalf.namePtr == nil { + return nil, ErrFuncPointerNil + } + nameValue, exists := table[x.EdgeHalf.namePtr] + if !exists { + return nil, ErrTableNoValue } // the edge name can be a single string or a list of strings... @@ -872,7 +905,7 @@ func (obj *StmtRes) edges(resName string) ([]*interfaces.Edge, error) { // metaparams is a helper function to set the metaparams that come from the // resource on to the individual resource we're working on. -func (obj *StmtRes) metaparams(res engine.Res) error { +func (obj *StmtRes) metaparams(table map[interfaces.Func]types.Value, res engine.Res) error { meta := engine.DefaultMetaParams.Copy() // defaults var rm *engine.ReversibleMeta @@ -895,9 +928,12 @@ func (obj *StmtRes) metaparams(res engine.Res) error { } if x.Condition != nil { - b, err := x.Condition.Value() - if err != nil { - return err + if x.conditionPtr == nil { + return ErrFuncPointerNil + } + b, exists := table[x.conditionPtr] + if !exists { + return ErrTableNoValue } if !b.Bool() { // if value exists, and is false, skip it @@ -905,9 +941,12 @@ func (obj *StmtRes) metaparams(res engine.Res) error { } } - v, err := x.MetaExpr.Value() - if err != nil { - return err + if x.metaExprPtr == nil { + return ErrFuncPointerNil + } + v, exists := table[x.metaExprPtr] + if !exists { + return ErrTableNoValue } switch p := strings.ToLower(x.Property); p { @@ -1075,9 +1114,11 @@ type StmtResContents interface { // StmtResField represents a single field in the parsed resource representation. // This does not satisfy the Stmt interface. type StmtResField struct { - Field string - Value interfaces.Expr - Condition interfaces.Expr // the value will be used if nil or true + Field string + Value interfaces.Expr + valuePtr interfaces.Func // ptr for table lookup + Condition interfaces.Expr // the value will be used if nil or true + conditionPtr interfaces.Func // ptr for table lookup } // String returns a short representation of this statement. @@ -1223,11 +1264,11 @@ func (obj *StmtResField) Ordering(produces map[string]interfaces.Node) (*pgraph. // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. func (obj *StmtResField) SetScope(scope *interfaces.Scope) error { - if err := obj.Value.SetScope(scope); err != nil { + if err := obj.Value.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } if obj.Condition != nil { - if err := obj.Condition.SetScope(scope); err != nil { + if err := obj.Condition.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } } @@ -1306,18 +1347,20 @@ func (obj *StmtResField) Graph() (*pgraph.Graph, error) { return nil, err } - g, err := obj.Value.Graph() + g, f, err := obj.Value.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.valuePtr = f if obj.Condition != nil { - g, err := obj.Condition.Graph() + g, f, err := obj.Condition.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.conditionPtr = f } return graph, nil @@ -1326,9 +1369,10 @@ func (obj *StmtResField) Graph() (*pgraph.Graph, error) { // StmtResEdge represents a single edge property in the parsed resource // representation. This does not satisfy the Stmt interface. type StmtResEdge struct { - Property string // TODO: iota constant instead? - EdgeHalf *StmtEdgeHalf - Condition interfaces.Expr // the value will be used if nil or true + Property string // TODO: iota constant instead? + EdgeHalf *StmtEdgeHalf + Condition interfaces.Expr // the value will be used if nil or true + conditionPtr interfaces.Func // ptr for table lookup } // String returns a short representation of this statement. @@ -1485,7 +1529,7 @@ func (obj *StmtResEdge) SetScope(scope *interfaces.Scope) error { return err } if obj.Condition != nil { - if err := obj.Condition.SetScope(scope); err != nil { + if err := obj.Condition.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } } @@ -1546,11 +1590,12 @@ func (obj *StmtResEdge) Graph() (*pgraph.Graph, error) { graph.AddGraph(g) if obj.Condition != nil { - g, err := obj.Condition.Graph() + g, f, err := obj.Condition.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.conditionPtr = f } return graph, nil @@ -1563,9 +1608,11 @@ func (obj *StmtResEdge) Graph() (*pgraph.Graph, error) { // correspond to the particular meta parameter specified. This does not satisfy // the Stmt interface. type StmtResMeta struct { - Property string // TODO: iota constant instead? - MetaExpr interfaces.Expr - Condition interfaces.Expr // the value will be used if nil or true + Property string // TODO: iota constant instead? + MetaExpr interfaces.Expr + metaExprPtr interfaces.Func // ptr for table lookup + Condition interfaces.Expr // the value will be used if nil or true + conditionPtr interfaces.Func // ptr for table lookup } // String returns a short representation of this statement. @@ -1736,11 +1783,11 @@ func (obj *StmtResMeta) Ordering(produces map[string]interfaces.Node) (*pgraph.G // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. func (obj *StmtResMeta) SetScope(scope *interfaces.Scope) error { - if err := obj.MetaExpr.SetScope(scope); err != nil { + if err := obj.MetaExpr.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } if obj.Condition != nil { - if err := obj.Condition.SetScope(scope); err != nil { + if err := obj.Condition.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } } @@ -1882,18 +1929,21 @@ func (obj *StmtResMeta) Graph() (*pgraph.Graph, error) { return nil, err } - g, err := obj.MetaExpr.Graph() + g, f, err := obj.MetaExpr.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.metaExprPtr = f if obj.Condition != nil { - g, err := obj.Condition.Graph() + g, f, err := obj.Condition.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.conditionPtr = f + } return graph, nil @@ -2155,13 +2205,17 @@ func (obj *StmtEdge) Graph() (*pgraph.Graph, error) { // called by this Output function if they are needed to produce the output. In // the case of this edge statement, this is definitely the case. This edge stmt // returns output consisting of edges. -func (obj *StmtEdge) Output() (*interfaces.Output, error) { +func (obj *StmtEdge) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { edges := []*interfaces.Edge{} + // EdgeHalfList goes in a chain, so we increment like i++ and not i+=2. for i := 0; i < len(obj.EdgeHalfList)-1; i++ { - nameValue1, err := obj.EdgeHalfList[i].Name.Value() - if err != nil { - return nil, err + if obj.EdgeHalfList[i].namePtr == nil { + return nil, ErrFuncPointerNil + } + nameValue1, exists := table[obj.EdgeHalfList[i].namePtr] + if !exists { + return nil, ErrTableNoValue } // the edge name can be a single string or a list of strings... @@ -2183,9 +2237,12 @@ func (obj *StmtEdge) Output() (*interfaces.Output, error) { return nil, fmt.Errorf("unhandled resource name type: %+v", nameValue1.Type()) } - nameValue2, err := obj.EdgeHalfList[i+1].Name.Value() - if err != nil { - return nil, err + if obj.EdgeHalfList[i+1].namePtr == nil { + return nil, ErrFuncPointerNil + } + nameValue2, exists := table[obj.EdgeHalfList[i+1].namePtr] + if !exists { + return nil, ErrTableNoValue } names2 := []string{} // list of names to build @@ -2233,6 +2290,7 @@ func (obj *StmtEdge) Output() (*interfaces.Output, error) { type StmtEdgeHalf struct { Kind string // kind of resource, eg: pkg, file, svc, etc... Name interfaces.Expr // unique name for the res of this kind + namePtr interfaces.Func // ptr for table lookup SendRecv string // name of field to send/recv from/to, empty to ignore } @@ -2306,7 +2364,7 @@ func (obj *StmtEdgeHalf) Copy() (*StmtEdgeHalf, error) { // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. func (obj *StmtEdgeHalf) SetScope(scope *interfaces.Scope) error { - return obj.Name.SetScope(scope) + return obj.Name.SetScope(scope, map[string]interfaces.Expr{}) } // Unify returns the list of invariants that this node produces. It recursively @@ -2367,7 +2425,12 @@ func (obj *StmtEdgeHalf) Unify() ([]interfaces.Invariant, error) { // no outgoing edges have produced at least a single value, then the resources // know they're able to be built. func (obj *StmtEdgeHalf) Graph() (*pgraph.Graph, error) { - return obj.Name.Graph() + g, f, err := obj.Name.Graph(make(map[string]interfaces.Func)) + if err != nil { + return nil, err + } + obj.namePtr = f + return g, nil } // StmtIf represents an if condition that contains between one and two branches @@ -2377,9 +2440,10 @@ func (obj *StmtEdgeHalf) Graph() (*pgraph.Graph, error) { // optional, it is the else branch, although this struct allows either to be // optional, even if it is not commonly used. type StmtIf struct { - Condition interfaces.Expr - ThenBranch interfaces.Stmt // optional, but usually present - ElseBranch interfaces.Stmt // optional + Condition interfaces.Expr + conditionPtr interfaces.Func // ptr for table lookup + ThenBranch interfaces.Stmt // optional, but usually present + ElseBranch interfaces.Stmt // optional } // String returns a short representation of this statement. @@ -2597,7 +2661,7 @@ func (obj *StmtIf) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph, // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. func (obj *StmtIf) SetScope(scope *interfaces.Scope) error { - if err := obj.Condition.SetScope(scope); err != nil { + if err := obj.Condition.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } if obj.ThenBranch != nil { @@ -2669,11 +2733,12 @@ func (obj *StmtIf) Graph() (*pgraph.Graph, error) { return nil, err } - g, err := obj.Condition.Graph() + g, f, err := obj.Condition.Graph(make(map[string]interfaces.Func)) if err != nil { return nil, err } graph.AddGraph(g) + obj.conditionPtr = f for _, x := range []interfaces.Stmt{obj.ThenBranch, obj.ElseBranch} { if x == nil { @@ -2693,20 +2758,24 @@ func (obj *StmtIf) Graph() (*pgraph.Graph, error) { // is used to build the output graph. This only exists for statements. The // analogous function for expressions is Value. Those Value functions might get // called by this Output function if they are needed to produce the output. -func (obj *StmtIf) Output() (*interfaces.Output, error) { - b, err := obj.Condition.Value() - if err != nil { - return nil, err +func (obj *StmtIf) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { + if obj.conditionPtr == nil { + return nil, ErrFuncPointerNil + } + b, exists := table[obj.conditionPtr] + if !exists { + return nil, ErrTableNoValue } var output *interfaces.Output + var err error if b.Bool() { // must not panic! if obj.ThenBranch != nil { // logically then branch is optional - output, err = obj.ThenBranch.Output() + output, err = obj.ThenBranch.Output(table) } } else { if obj.ElseBranch != nil { // else branch is optional - output, err = obj.ElseBranch.Output() + output, err = obj.ElseBranch.Output(table) } } if err != nil { @@ -2731,8 +2800,9 @@ func (obj *StmtIf) Output() (*interfaces.Output, error) { // the bind statement's are correctly applied in this scope, and irrespective of // their order of definition. type StmtProg struct { - data *interfaces.Data - scope *interfaces.Scope // store for use by imports + data *interfaces.Data + scope *interfaces.Scope // store for use by imports + importedVars map[string]interfaces.Expr // store for use by Graph // TODO: should this be a map? if so, how would we sort it to loop it? importProgs []*StmtProg // list of child programs after running SetScope @@ -3340,6 +3410,11 @@ func (obj *StmtProg) importScopeWithInputs(s string, scope *interfaces.Scope, pa if prog.scope.IsEmpty() { return nil, fmt.Errorf("could not find any non-empty scope") } + fmt.Printf("imported scope:\n") + for k, v := range prog.scope.Variables { + // print the type of v + fmt.Printf(" %s: %T\n", k, v) + } // save a reference to the prog for future usage in Unify/Graph/Etc... obj.importProgs = append(obj.importProgs, prog) @@ -3366,6 +3441,7 @@ func (obj *StmtProg) importScopeWithInputs(s string, scope *interfaces.Scope, pa // args. func (obj *StmtProg) SetScope(scope *interfaces.Scope) error { newScope := scope.Copy() + obj.importedVars = make(map[string]interfaces.Expr) // start by looking for any `import` statements to pull into the scope! // this will run child lexing/parsing, interpolation, and scope setting @@ -3422,6 +3498,7 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error { } newVariables[newName] = imp.Name newScope.Variables[newName] = x // merge + obj.importedVars[newName] = x } for name, x := range importedScope.Functions { newName := alias + interfaces.ModuleSep + name @@ -3468,7 +3545,10 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error { binds[bind.Ident] = struct{}{} // mark as found in scope // add to scope, (overwriting, aka shadowing is ok) - newScope.Variables[bind.Ident] = bind.Value + newScope.Variables[bind.Ident] = &ExprPoly{ + Definition: bind.Value, + CapturedScope: newScope, + } if obj.data.Debug { // TODO: is this message ever useful? obj.data.Logf("prog: set scope: bind collect: (%+v): %+v (%T) is %p", bind.Ident, bind.Value, bind.Value, bind.Value) } @@ -3504,7 +3584,10 @@ func (obj *StmtProg) SetScope(scope *interfaces.Scope) error { if len(fnList) == 1 { fn := fnList[0].Func // local reference to avoid changing it in the loop... // add to scope, (overwriting, aka shadowing is ok) - newScope.Functions[name] = fn // store the *ExprFunc + newScope.Functions[name] = &ExprPoly{ + Definition: fn, // store the *ExprFunc + CapturedScope: newScope, + } continue } @@ -3757,13 +3840,13 @@ func (obj *StmtProg) Graph() (*pgraph.Graph, error) { } // add graphs from SetScope's imported child programs - for _, x := range obj.importProgs { - g, err := x.Graph() - if err != nil { - return nil, err - } - graph.AddGraph(g) - } + //for _, x := range obj.importProgs { + // g, err := x.Graph(env) + // if err != nil { + // return nil, err + // } + // graph.AddGraph(g) + //} return graph, nil } @@ -3772,7 +3855,7 @@ func (obj *StmtProg) Graph() (*pgraph.Graph, error) { // is used to build the output graph. This only exists for statements. The // analogous function for expressions is Value. Those Value functions might get // called by this Output function if they are needed to produce the output. -func (obj *StmtProg) Output() (*interfaces.Output, error) { +func (obj *StmtProg) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { resources := []engine.Res{} edges := []*interfaces.Edge{} @@ -3792,7 +3875,7 @@ func (obj *StmtProg) Output() (*interfaces.Output, error) { continue } - output, err := stmt.Output() + output, err := stmt.Output(table) if err != nil { return nil, err } @@ -3957,7 +4040,7 @@ func (obj *StmtFunc) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap // necessary in order to reach this, in particular in situations when a bound // expression points to a previously bound expression. func (obj *StmtFunc) SetScope(scope *interfaces.Scope) error { - return obj.Func.SetScope(scope) + return obj.Func.SetScope(scope, map[string]interfaces.Expr{}) } // Unify returns the list of invariants that this node produces. It recursively @@ -3982,13 +4065,13 @@ func (obj *StmtFunc) Unify() ([]interfaces.Invariant, error) { // children might. This particular func statement adds its linked expression to // the graph. func (obj *StmtFunc) Graph() (*pgraph.Graph, error) { - //return obj.Func.Graph() // nope! + //return obj.Func.Graph(nil) // nope! return pgraph.NewGraph("stmtfunc") // do this in ExprCall instead } // Output for the func statement produces no output. Any values of interest come // from the use of the func which this binds the function to. -func (obj *StmtFunc) Output() (*interfaces.Output, error) { +func (obj *StmtFunc) Output(map[interfaces.Func]types.Value) (*interfaces.Output, error) { return interfaces.EmptyOutput(), nil } @@ -4130,7 +4213,7 @@ func (obj *StmtClass) SetScope(scope *interfaces.Scope) error { scope = interfaces.EmptyScope() } obj.scope = scope // store for later - return obj.Body.SetScope(scope) + return nil } // Unify returns the list of invariants that this node produces. It recursively @@ -4160,8 +4243,8 @@ func (obj *StmtClass) Graph() (*pgraph.Graph, error) { // come from the use of the include which this binds the statements to. This is // usually called from the parent in StmtProg, but it skips running it so that // it can be called from the StmtInclude Output method. -func (obj *StmtClass) Output() (*interfaces.Output, error) { - return obj.Body.Output() +func (obj *StmtClass) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { + return obj.Body.Output(table) } // StmtInclude causes a user defined class to get used. It's effectively the way @@ -4364,7 +4447,7 @@ func (obj *StmtInclude) SetScope(scope *interfaces.Scope) error { // make sure to propagate the scope to our input args! if obj.Args != nil { for _, x := range obj.Args { - if err := x.SetScope(scope); err != nil { + if err := x.SetScope(scope, map[string]interfaces.Expr{}); err != nil { return err } } @@ -4421,7 +4504,7 @@ func (obj *StmtInclude) SetScope(scope *interfaces.Scope) error { // need to use the original scope of the class as it was set as the // basis for this scope, so that we overwrite it only with the arg // changes. - if err := obj.class.SetScope(newScope); err != nil { + if err := obj.class.Body.SetScope(newScope); err != nil { return err } @@ -4505,8 +4588,8 @@ func (obj *StmtInclude) Graph() (*pgraph.Graph, error) { // called by this Output function if they are needed to produce the output. The // ultimate source of this output comes from the previously defined StmtClass // which should be found in our scope. -func (obj *StmtInclude) Output() (*interfaces.Output, error) { - return obj.class.Output() +func (obj *StmtInclude) Output(table map[interfaces.Func]types.Value) (*interfaces.Output, error) { + return obj.class.Output(table) } // StmtImport adds the exported scope definitions of a module into the current @@ -4586,7 +4669,7 @@ func (obj *StmtImport) Unify() ([]interfaces.Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This particular statement just returns an empty graph. func (obj *StmtImport) Graph() (*pgraph.Graph, error) { - return pgraph.NewGraph("import") + return pgraph.NewGraph("import") // empty graph } // Output returns the output that this include produces. This output is what is @@ -4595,7 +4678,7 @@ func (obj *StmtImport) Graph() (*pgraph.Graph, error) { // called by this Output function if they are needed to produce the output. This // import statement itself produces no output, as it is only used to populate // the scope so that others can use that to produce values and output. -func (obj *StmtImport) Output() (*interfaces.Output, error) { +func (obj *StmtImport) Output(map[interfaces.Func]types.Value) (*interfaces.Output, error) { return interfaces.EmptyOutput(), nil } @@ -4672,15 +4755,11 @@ func (obj *StmtComment) Unify() ([]interfaces.Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This particular graph does nothing clever. func (obj *StmtComment) Graph() (*pgraph.Graph, error) { - graph, err := pgraph.NewGraph("comment") - if err != nil { - return nil, err - } - return graph, nil + return pgraph.NewGraph("comment") } // Output for the comment statement produces no output. -func (obj *StmtComment) Output() (*interfaces.Output, error) { +func (obj *StmtComment) Output(map[interfaces.Func]types.Value) (*interfaces.Output, error) { return interfaces.EmptyOutput(), nil } @@ -4737,7 +4816,7 @@ func (obj *ExprBool) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap // SetScope does nothing for this struct, because it has no child nodes, and it // does not need to know about the parent scope. It does however store it for // later possible use. -func (obj *ExprBool) SetScope(scope *interfaces.Scope) error { +func (obj *ExprBool) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } @@ -4767,26 +4846,30 @@ func (obj *ExprBool) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprBool) Func() (interfaces.Func, error) { + return &structs.ConstFunc{ + Value: &types.BoolValue{V: obj.V}, + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr // interface directly produce vertices (and possible children) where as nodes // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it. -func (obj *ExprBool) Graph() (*pgraph.Graph, error) { +func (obj *ExprBool) Graph(map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("bool") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprBool) Func() (interfaces.Func, error) { - return &structs.ConstFunc{ - Value: &types.BoolValue{V: obj.V}, - }, nil + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) + return graph, function, nil } // SetValue for a bool expression is always populated statically, and does not @@ -4913,7 +4996,7 @@ func (obj *ExprStr) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph // SetScope does nothing for this struct, because it has no child nodes, and it // does not need to know about the parent scope. It does however store it for // later possible use. -func (obj *ExprStr) SetScope(scope *interfaces.Scope) error { +func (obj *ExprStr) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } @@ -4943,26 +5026,30 @@ func (obj *ExprStr) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprStr) Func() (interfaces.Func, error) { + return &structs.ConstFunc{ + Value: &types.StrValue{V: obj.V}, + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr // interface directly produce vertices (and possible children) where as nodes // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it. -func (obj *ExprStr) Graph() (*pgraph.Graph, error) { +func (obj *ExprStr) Graph(map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("str") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprStr) Func() (interfaces.Func, error) { - return &structs.ConstFunc{ - Value: &types.StrValue{V: obj.V}, - }, nil + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) + return graph, function, nil } // SetValue for an str expression is always populated statically, and does not @@ -5039,7 +5126,7 @@ func (obj *ExprInt) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph // SetScope does nothing for this struct, because it has no child nodes, and it // does not need to know about the parent scope. It does however store it for // later possible use. -func (obj *ExprInt) SetScope(scope *interfaces.Scope) error { +func (obj *ExprInt) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } @@ -5069,26 +5156,30 @@ func (obj *ExprInt) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprInt) Func() (interfaces.Func, error) { + return &structs.ConstFunc{ + Value: &types.IntValue{V: obj.V}, + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr // interface directly produce vertices (and possible children) where as nodes // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it. -func (obj *ExprInt) Graph() (*pgraph.Graph, error) { +func (obj *ExprInt) Graph(map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("int") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprInt) Func() (interfaces.Func, error) { - return &structs.ConstFunc{ - Value: &types.IntValue{V: obj.V}, - }, nil + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) + return graph, function, nil } // SetValue for an int expression is always populated statically, and does not @@ -5167,7 +5258,7 @@ func (obj *ExprFloat) Ordering(produces map[string]interfaces.Node) (*pgraph.Gra // SetScope does nothing for this struct, because it has no child nodes, and it // does not need to know about the parent scope. It does however store it for // later possible use. -func (obj *ExprFloat) SetScope(scope *interfaces.Scope) error { +func (obj *ExprFloat) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } @@ -5197,26 +5288,30 @@ func (obj *ExprFloat) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprFloat) Func() (interfaces.Func, error) { + return &structs.ConstFunc{ + Value: &types.FloatValue{V: obj.V}, + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr // interface directly produce vertices (and possible children) where as nodes // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it. -func (obj *ExprFloat) Graph() (*pgraph.Graph, error) { +func (obj *ExprFloat) Graph(map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("float") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprFloat) Func() (interfaces.Func, error) { - return &structs.ConstFunc{ - Value: &types.FloatValue{V: obj.V}, - }, nil + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) + return graph, function, nil } // SetValue for a float expression is always populated statically, and does not @@ -5370,14 +5465,14 @@ func (obj *ExprList) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. -func (obj *ExprList) SetScope(scope *interfaces.Scope) error { +func (obj *ExprList) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } obj.scope = scope for _, x := range obj.Elements { - if err := x.SetScope(scope); err != nil { + if err := x.SetScope(scope, context); err != nil { return err } } @@ -5505,6 +5600,20 @@ func (obj *ExprList) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprList) Func() (interfaces.Func, error) { + typ, err := obj.Type() + if err != nil { + return nil, err + } + + // composite func (list, map, struct) + return &structs.CompositeFunc{ + Type: typ, + Len: len(obj.Elements), + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr @@ -5512,49 +5621,31 @@ func (obj *ExprList) Unify() ([]interfaces.Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it, and // the edges from all of the child graphs to this. -func (obj *ExprList) Graph() (*pgraph.Graph, error) { +func (obj *ExprList) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("list") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) // each list element needs to point to the final list expression for index, x := range obj.Elements { // list elements in order - g, err := x.Graph() + g, f, err := x.Graph(env) if err != nil { - return nil, err + return nil, nil, err } + graph.AddGraph(g) fieldName := fmt.Sprintf("%d", index) // argNames as integers! edge := &interfaces.FuncEdge{Args: []string{fieldName}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for list, index `%d` was called twice", index)) - } - once = true - return edge - } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // element -> list - } - - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprList) Func() (interfaces.Func, error) { - typ, err := obj.Type() - if err != nil { - return nil, err + graph.AddEdge(f, function, edge) // element -> list } - // composite func (list, map, struct) - return &structs.CompositeFunc{ - Type: typ, - Len: len(obj.Elements), - }, nil + return graph, function, nil } // SetValue here is a no-op, because algorithmically when this is called from @@ -5803,17 +5894,17 @@ func (obj *ExprMap) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. -func (obj *ExprMap) SetScope(scope *interfaces.Scope) error { +func (obj *ExprMap) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } obj.scope = scope for _, x := range obj.KVs { - if err := x.Key.SetScope(scope); err != nil { + if err := x.Key.SetScope(scope, context); err != nil { return err } - if err := x.Val.SetScope(scope); err != nil { + if err := x.Val.SetScope(scope, context); err != nil { return err } } @@ -5981,6 +6072,20 @@ func (obj *ExprMap) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprMap) Func() (interfaces.Func, error) { + typ, err := obj.Type() + if err != nil { + return nil, err + } + + // composite func (list, map, struct) + return &structs.CompositeFunc{ + Type: typ, // the key/val types are known via this type + Len: len(obj.KVs), + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr @@ -5988,69 +6093,45 @@ func (obj *ExprMap) Unify() ([]interfaces.Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it, and // the edges from all of the child graphs to this. -func (obj *ExprMap) Graph() (*pgraph.Graph, error) { +func (obj *ExprMap) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("map") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) // each map key value pair needs to point to the final map expression for index, x := range obj.KVs { // map fields in order - g, err := x.Key.Graph() + g, f, err := x.Key.Graph(env) if err != nil { - return nil, err + return nil, nil, err } + graph.AddGraph(g) + // do the key names ever change? -- yes fieldName := fmt.Sprintf("key:%d", index) // stringify map key edge := &interfaces.FuncEdge{Args: []string{fieldName}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for map, key `%s` was called twice", fieldName)) - } - once = true - return edge - } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // key -> func + graph.AddEdge(f, function, edge) // key -> map } // each map key value pair needs to point to the final map expression for index, x := range obj.KVs { // map fields in order - g, err := x.Val.Graph() + g, f, err := x.Val.Graph(env) if err != nil { - return nil, err + return nil, nil, err } + graph.AddGraph(g) + fieldName := fmt.Sprintf("val:%d", index) // stringify map val edge := &interfaces.FuncEdge{Args: []string{fieldName}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for map, val `%s` was called twice", fieldName)) - } - once = true - return edge - } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // val -> func - } - - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprMap) Func() (interfaces.Func, error) { - typ, err := obj.Type() - if err != nil { - return nil, err + graph.AddEdge(f, function, edge) // val -> map } - // composite func (list, map, struct) - return &structs.CompositeFunc{ - Type: typ, // the key/val types are known via this type - Len: len(obj.KVs), - }, nil + return graph, function, nil } // SetValue here is a no-op, because algorithmically when this is called from @@ -6288,14 +6369,14 @@ func (obj *ExprStruct) Ordering(produces map[string]interfaces.Node) (*pgraph.Gr // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. -func (obj *ExprStruct) SetScope(scope *interfaces.Scope) error { +func (obj *ExprStruct) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } obj.scope = scope for _, x := range obj.Fields { - if err := x.Value.SetScope(scope); err != nil { + if err := x.Value.SetScope(scope, context); err != nil { return err } } @@ -6393,55 +6474,51 @@ func (obj *ExprStruct) Unify() ([]interfaces.Invariant, error) { return invariants, nil } -// Graph returns the reactive function graph which is expressed by this node. It -// includes any vertices produced by this node, and the appropriate edges to any -// vertices that are produced by its children. Nodes which fulfill the Expr -// interface directly produce vertices (and possible children) where as nodes -// that fulfill the Stmt interface do not produces vertices, where as their +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprStruct) Func() (interfaces.Func, error) { + typ, err := obj.Type() + if err != nil { + return nil, err + } + + // composite func (list, map, struct) + return &structs.CompositeFunc{ + Type: typ, + }, nil +} + +// Graph returns the reactive function graph which is expressed by this node. It +// includes any vertices produced by this node, and the appropriate edges to any +// vertices that are produced by its children. Nodes which fulfill the Expr +// interface directly produce vertices (and possible children) where as nodes +// that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it, and // the edges from all of the child graphs to this. -func (obj *ExprStruct) Graph() (*pgraph.Graph, error) { +func (obj *ExprStruct) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("struct") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) // each struct field needs to point to the final struct expression for _, x := range obj.Fields { // struct fields in order - g, err := x.Value.Graph() + g, f, err := x.Value.Graph(env) if err != nil { - return nil, err + return nil, nil, err } + graph.AddGraph(g) fieldName := x.Name edge := &interfaces.FuncEdge{Args: []string{fieldName}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for struct, arg `%s` was called twice", fieldName)) - } - once = true - return edge - } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // arg -> func - } - - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprStruct) Func() (interfaces.Func, error) { - typ, err := obj.Type() - if err != nil { - return nil, err + graph.AddEdge(f, function, edge) // field -> struct } - // composite func (list, map, struct) - return &structs.CompositeFunc{ - Type: typ, - }, nil + return graph, function, nil } // SetValue here is a no-op, because algorithmically when this is called from @@ -6517,17 +6594,14 @@ type ExprStructField struct { } // ExprFunc is a representation of a function value. This is not a function -// call, that is represented by ExprCall. This can represent either the contents -// of a StmtFunc, a lambda function, or a core system function. You may only use -// one of the internal representations of a function to build this, if you use -// more than one then the behaviour is not defined, and could conceivably panic. -// The first possibility is to specify the function via the Args, Return, and -// Body fields. This is used for native mcl code. The second possibility is to -// specify the function via the Function field only. This is used for built-in -// functions that implement the Func API. The third possibility is to specify a -// list of function values via the Values field. This is used for built-in -// functions that implement the simple function API or the simplepoly function -// API and that aren't wrapped in the Func API. (This was the historical case.) +// call, that is represented by ExprCall. +// +// There are several kinds of functions which can be represented: +// 1. The contents of a StmtFunc (set Args, Return, and Body) +// 2. A lambda function (also set Args, Return, and Body) +// 3. A stateful built-in function (set Function) +// 4. A pure built-in function (set Values to a singleton) +// 5. A pure polymorphic built-in function (set Values to a list) type ExprFunc struct { data *interfaces.Data scope *interfaces.Scope // store for referencing this later @@ -6543,6 +6617,12 @@ type ExprFunc struct { // This can include a string name and a type, however the type might be // absent here. Args []*interfaces.Arg + + // One ExprParam is created for each parameter, and the ExprVars which + // refer to those parameters are set to point to the corresponding + // ExprParam. + params []*ExprParam + // Return is the return type of the function if it was defined. Return *types.Type // return type if specified // Body is the contents of the function. It can be any expression. @@ -6558,7 +6638,7 @@ type ExprFunc struct { Values []*types.FuncValue // XXX: is this necessary? - V func([]types.Value) (types.Value, error) + //V func(interfaces.Txn, []pgraph.Vertex) (pgraph.Vertex, error) } // String returns a short representation of this expression. @@ -6696,7 +6776,6 @@ func (obj *ExprFunc) Interpolate() (interfaces.Expr, error) { Function: obj.Function, function: obj.function, Values: obj.Values, - V: obj.V, }, nil } @@ -6727,6 +6806,20 @@ func (obj *ExprFunc) Copy() (interfaces.Expr, error) { var function interfaces.Func if obj.Function != nil { function = obj.Function() // force re-build a new pointer here! + // restore the type we previously set in SetType() + if obj.typ != nil { + polyFn, ok := function.(interfaces.PolyFunc) // is it statically polymorphic? + if ok { + newTyp, err := polyFn.Build(obj.typ) + if err != nil { + return nil, errwrap.Wrapf(err, "could not build expr func") + } + // Cmp doesn't compare arg names. Check it's compatible... + if err := obj.typ.Cmp(newTyp); err != nil { + return nil, errwrap.Wrapf(err, "incompatible type") + } + } + } // pass in some data to the function // TODO: do we want to pass in the full obj.data instead ? if dataFunc, ok := function.(interfaces.DataFunc); ok { @@ -6760,7 +6853,6 @@ func (obj *ExprFunc) Copy() (interfaces.Expr, error) { Function: obj.Function, function: function, Values: obj.Values, // XXX: do we need to force rebuild these? - V: obj.V, }, nil } @@ -6809,48 +6901,29 @@ func (obj *ExprFunc) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. -func (obj *ExprFunc) SetScope(scope *interfaces.Scope) error { - // TODO: Should we merge the existing obj.scope with the new one? This - // gets called multiple times, maybe doing that would simplify other - // parts of the code. +func (obj *ExprFunc) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } obj.scope = scope // store for later if obj.Body != nil { - newScope := scope.Copy() - - if obj.data.Debug { - if obj.Title != "" { - obj.data.Logf("func: %s: scope: pull index 0", obj.Title) - } else { - obj.data.Logf("func: scope: pull index 0") - } + bodyContext := make(map[string]interfaces.Expr) + for k, v := range context { + bodyContext[k] = v } - indexes, exists := newScope.PullIndexes() - if exists { - if i, j := len(indexes), len(obj.Args); i != j { - return fmt.Errorf("called with %d args, but function requires %d", i, j) - } - // this version is more future proof, but less logical... - // in particular, if there are no indices, then this is skipped! - for i, arg := range indexes { // unrename - name := obj.Args[i].Name - newScope.Variables[name] = arg - } - // this version is less future proof, but more logical... - //for i, arg := range obj.Args { // copy (unrename) - // newScope.Variables[arg.Name] = indexes[i] - //} + // add the parameters to the context + // make a list as long as obj.Args + obj.params = make([]*ExprParam, len(obj.Args)) + for i, arg := range obj.Args { + param := &ExprParam{Name: arg.Name, Typ: arg.Type} + obj.params[i] = param + bodyContext[arg.Name] = param } - // We used to store newScope here as bodyScope for later lookup! - //obj.bodyScope = newScope // store for later - // Instead we just added a private getScope method for expr's... - if err := obj.Body.SetScope(newScope); err != nil { - return err + if err := obj.Body.SetScope(scope, bodyContext); err != nil { + return errwrap.Wrapf(err, "failed to set scope on function body") } } @@ -6878,9 +6951,12 @@ func (obj *ExprFunc) SetType(typ *types.Type) error { if obj.Function != nil { polyFn, ok := obj.function.(interfaces.PolyFunc) // is it statically polymorphic? if ok { - if err := polyFn.Build(typ); err != nil { + newTyp, err := polyFn.Build(typ) + if err != nil { return errwrap.Wrapf(err, "could not build expr func") } + // Cmp doesn't compare arg names. + typ = newTyp // check it's compatible down below... } } @@ -6891,7 +6967,7 @@ func (obj *ExprFunc) SetType(typ *types.Type) error { return errwrap.Wrapf(err, "could not build values func") } // TODO: build the function here for later use if that is wanted - //fn := obj.Values[index].Copy().(*types.FuncValue) + //fn := obj.Values[index].Copy() //fn.T = typ.Copy() // overwrites any contained "variant" type } @@ -6925,6 +7001,11 @@ func (obj *ExprFunc) Type() (*types.Type, error) { } if obj.Function != nil { + if obj.function == nil { + // TODO: should we return ErrTypeCurrentlyUnknown instead? + panic("unexpected empty function") + //return nil, interfaces.ErrTypeCurrentlyUnknown + } sig := obj.function.Info().Sig if sig != nil && !sig.HasVariant() && obj.typ == nil { // type is now known statically return sig, nil @@ -7011,105 +7092,25 @@ func (obj *ExprFunc) Unify() ([]interfaces.Invariant, error) { // collect all the invariants of the body if obj.Body != nil { - invars, err := obj.Body.Unify() - if err != nil { - return nil, err + expr2Ord := []string{} + expr2Map := map[string]interfaces.Expr{} + for i, arg := range obj.Args { + expr2Ord = append(expr2Ord, arg.Name) + expr2Map[arg.Name] = obj.params[i] } - invariants = append(invariants, invars...) - } + funcInvariant := &interfaces.EqualityWrapFuncInvariant{ + Expr1: obj, + Expr2Map: expr2Map, + Expr2Ord: expr2Ord, + Expr2Out: obj.Body, + } + invariants = append(invariants, funcInvariant) - if obj.Body != nil { // XXX: nuke this section? invars, err := obj.Body.Unify() if err != nil { return nil, err } invariants = append(invariants, invars...) - - mapped := make(map[string]interfaces.Expr) - ordered := []string{} - - // If the args are passed in by index, then we can use this, - // otherwise we can try and look them up in the standard scope. - if indexes, exists := obj.scope.Indexes[0]; exists { - if i, j := len(indexes), len(obj.Args); i != j { - return nil, fmt.Errorf("called with %d args, but function requires %d", i, j) - } - // this version is more future proof, but less logical... - // in particular, if there are no indices, then this is skipped! - for i, arg := range indexes { // unrename - name := obj.Args[i].Name - mapped[name] = arg - ordered = append(ordered, name) - - // if the arg's type is known statically... - if typ := obj.Args[i].Type; typ != nil { - invar := &interfaces.EqualsInvariant{ - Expr: arg, - Type: typ, - } - invariants = append(invariants, invar) - } - - // The scope that is built for the body, should - // have variables that correspond to the inputs. - bodyScope, err := getScope(obj.Body) - if err != nil { - // programming error? - return nil, errwrap.Wrapf(err, "can't get body scope") - } - if bodyScope != nil { // TODO: can this be nil? - invar := &interfaces.EqualityInvariant{ - Expr1: arg, - Expr2: bodyScope.Variables[name], - } - invariants = append(invariants, invar) - } - } - - } else { - // XXX: i don't think this branch is ever used... - return nil, fmt.Errorf("unexpected branch") - //for _, arg := range obj.Args { - // expr, exists := obj.scope.Variables[arg.Name] - // if !exists { - // // programming error ? - // return nil, fmt.Errorf("expected arg `%s` was missing from scope", arg.Name) - // } - // mapped[arg.Name] = expr - // ordered = append(ordered, arg.Name) - // - // // if the arg's type is known statically... - // if typ := arg.Type; typ != nil { - // invar := &interfaces.EqualsInvariant{ - // Expr: expr, - // Type: typ, - // } - // invariants = append(invariants, invar) - // } - // - // // TODO: do we need to add something like this? - // //bodyScope, err := getScope(obj.Body) - // //if err != nil { - // // // programming error? - // // return nil, errwrap.Wrapf(err, "can't get body scope") - // //} - // //// The scoped variable should match the arg. - // //invar := &interfaces.EqualityInvariant{ - // // Expr1: expr, - // // Expr2: bodyScope.Variables[name], // ??? - // //} - // //invariants = append(invariants, invar) - //} - } - - // XXX: is this the right kind of invariant??? - invariant := &interfaces.EqualityWrapFuncInvariant{ - Expr1: obj, - Expr2Map: mapped, - Expr2Ord: ordered, - Expr2Out: obj.Body, - } - invariants = append(invariants, invariant) } // return type must be equal to the body expression @@ -7211,107 +7212,124 @@ func (obj *ExprFunc) Unify() ([]interfaces.Invariant, error) { // interface directly produce vertices (and possible children) where as nodes // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it. -func (obj *ExprFunc) Graph() (*pgraph.Graph, error) { - graph, err := pgraph.NewGraph("func") - if err != nil { - return nil, err - } - graph.AddVertex(obj) +func (obj *ExprFunc) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { + // This implementation produces a graph with a single node of in-degree + // zero which outputs a single FuncValue. The FuncValue is a closure, in + // that it holds both a lambda body and a captured environment. This + // environment, which we receive from the caller, gives information + // about the variables declared _outside_ of the lambda, at the time the + // lambda is returned. + // + // Each time the FuncValue is called, it produces a separate graph, the + // subgraph which computes the lambda's output value from the lambda's + // argument values. The nodes created for that subgraph have a shorter + // life span than the nodes in the captured environment. + + //graph, err := pgraph.NewGraph("func") + //if err != nil { + // return nil, nil, err + //} + //function, err := obj.Func() + //if err != nil { + // return nil, nil, err + //} + //graph.AddVertex(function) + var funcValueFunc interfaces.Func if obj.Body != nil { - g, err := obj.Body.Graph() - if err != nil { - return nil, err - } + funcValueFunc = structs.FuncValueToConstFunc(&full.FuncValue{ + V: func(innerTxn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + // Extend the environment with the arguments. + extendedEnv := make(map[string]interfaces.Func) + for k, v := range env { + extendedEnv[k] = v + } + for i, arg := range obj.Args { + extendedEnv[arg.Name] = args[i] + } - // We need to add this edge, because if this isn't linked, then - // when we add an edge from this, then we'll get two because the - // contents aren't linked. - name := "body" // TODO: what should we name this? - edge := &interfaces.FuncEdge{Args: []string{name}} + // Create a subgraph from the lambda's body, instantiating the + // lambda's parameters with the args and the other variables + // with the nodes in the captured environment. + subgraph, bodyFunc, err := obj.Body.Graph(extendedEnv) + if err != nil { + return nil, errwrap.Wrapf(err, "could not create the lambda body's subgraph") + } - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for func was called twice")) - } - once = true - return edge + innerTxn.AddGraph(subgraph) + + return bodyFunc, nil + }, + T: obj.typ, + }) + } else if obj.Function != nil { + // obj.function is a node which transforms input values into + // an output value, but we need to construct a node which takes no + // inputs and produces a FuncValue, so we need to wrap it. + + funcValueFunc = structs.FuncValueToConstFunc(&full.FuncValue{ + V: func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + // Copy obj.function so that the underlying ExprFunc.function gets + // refreshed with a new ExprFunc.Function() call. Otherwise, multiple + // calls to this function will share the same Func. + exprCopy, err := obj.Copy() + if err != nil { + return nil, errwrap.Wrapf(err, "could not copy expression") + } + funcExprCopy, ok := exprCopy.(*ExprFunc) + if !ok { + // programming error + return nil, errwrap.Wrapf(err, "ExprFunc.Copy() does not produce an ExprFunc") + } + valueTransformingFunc := funcExprCopy.function + txn.AddVertex(valueTransformingFunc) + for i, arg := range args { + argName := obj.typ.Ord[i] + txn.AddEdge(arg, valueTransformingFunc, &interfaces.FuncEdge{ + Args: []string{argName}, + }) + } + return valueTransformingFunc, nil + }, + T: obj.typ, + }) + } else /* len(obj.Values) > 0 */ { + index, err := langutil.FnMatch(obj.typ, obj.Values) + if err != nil { + // programming error + // since type checking succeeded at this point, there should only be one match + return nil, nil, errwrap.Wrapf(err, "multiple matches found") } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // body -> func - } + simpleFn := obj.Values[index] + simpleFn.T = obj.typ - if obj.Function != nil { // no input args are needed, func is built-in. - // TODO: is there anything to do ? + funcValueFunc = structs.SimpleFnToConstFunc(fmt.Sprintf("title: %s", obj.Title), simpleFn) } - if len(obj.Values) > 0 { // no input args are needed, func is built-in. - // TODO: is there anything to do ? - } - - return graph, nil -} -// Func returns the reactive stream of values that this expression produces. We -// need this indirection, because our returned function that actually runs also -// accepts the "body" of the function (an expr) as an input. -func (obj *ExprFunc) Func() (interfaces.Func, error) { - typ, err := obj.Type() + outerGraph, err := pgraph.NewGraph("ExprFunc") if err != nil { - return nil, err - } - - if obj.Body != nil { - // TODO: i think this is unused - //f, err := obj.Body.Func() - //if err != nil { - // return nil, err - //} - - // direct func - return &structs.FunctionFunc{ - Type: typ, // this is a KindFunc - //Func: f, - Edge: "body", // the edge name used above in Graph is this... - }, nil - } - - if obj.Function != nil { - // XXX: is this correct? - return &structs.FunctionFunc{ - Type: typ, // this is a KindFunc - Func: obj.function, // pass it through - Edge: "", // no edge, since nothing is incoming to the built-in - }, nil + return nil, nil, err } - - // third kind - //if len(obj.Values) > 0 - index, err := langutil.FnMatch(typ, obj.Values) - if err != nil { - // programming error ? - return nil, errwrap.Wrapf(err, "no valid function found") - } - // build - // TODO: this could probably be done in SetType and cached in the struct - fn := obj.Values[index].Copy().(*types.FuncValue) - fn.T = typ.Copy() // overwrites any contained "variant" type - - return &structs.FunctionFunc{ - Type: typ, // this is a KindFunc - Fn: fn, // pass it through - Edge: "", // no edge, since nothing is incoming to the built-in - }, nil + outerGraph.AddVertex(funcValueFunc) + return outerGraph, funcValueFunc, nil } // SetValue for a func expression is always populated statically, and does not // ever receive any incoming values (no incoming edges) so this should never be // called. It has been implemented for uniformity. func (obj *ExprFunc) SetValue(value types.Value) error { - if err := obj.typ.Cmp(value.Type()); err != nil { - return err - } - // FIXME: is this part necessary? - obj.V = value.Func() + // We don't need to do anything because no resource has a function field and + // so nobody is going to call Value(). + + //if err := obj.typ.Cmp(value.Type()); err != nil { + // return err + //} + //// FIXME: is this part necessary? + //funcValue, worked := value.(*full.FuncValue) + //if !worked { + // return fmt.Errorf("expected a FuncValue") + //} + //obj.V = funcValue.V return nil } @@ -7320,11 +7338,12 @@ func (obj *ExprFunc) SetValue(value types.Value) error { // This might get called speculatively (early) during unification to learn more. // This particular value is always known since it is a constant. func (obj *ExprFunc) Value() (types.Value, error) { - // TODO: implement speculative value lookup (if not already sufficient) - return &types.FuncValue{ - V: obj.V, - T: obj.typ, - }, nil + panic("ExprFunc does not store its latest value because resources are not expected to have function fields.") + //// TODO: implement speculative value lookup (if not already sufficient) + //return &full.FuncValue{ + // V: obj.V, + // T: obj.typ, + //}, nil } // ExprCall is a representation of a function call. This does not represent the @@ -7533,7 +7552,7 @@ func (obj *ExprCall) Ordering(produces map[string]interfaces.Node) (*pgraph.Grap // which it propagates this downwards to. This particular function has been // heavily optimized to work correctly with calling functions with the correct // args. Edit cautiously and with extensive testing. -func (obj *ExprCall) SetScope(scope *interfaces.Scope) error { +func (obj *ExprCall) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } @@ -7543,147 +7562,75 @@ func (obj *ExprCall) SetScope(scope *interfaces.Scope) error { obj.data.Logf("call: %s(%t): scope: functions: %+v", obj.Name, obj.Var, obj.scope.Functions) } - // Remember that we *want* to propagate this scope into the args that - // we use, but we DON'T want to propagate it into the function body... - // Only the args should get propagated into it that way. + // scope-check the arguments for _, x := range obj.Args { - if err := x.SetScope(scope); err != nil { + if err := x.SetScope(scope, context); err != nil { return err } } - // which scope should we look in for our function? - var funcScope map[string]interfaces.Expr + var prefixedName string + var target interfaces.Expr if obj.Var { - funcScope = obj.scope.Variables // lambda value + // The call looks like $f(). + prefixedName = "$" + obj.Name + if f, exists := context[obj.Name]; exists { + // $f refers to a parameter bound by an enclosing lambda definition. + target = f + } else { + f, exists := obj.scope.Variables[obj.Name] + if !exists { + return fmt.Errorf("func `%s` does not exist in this scope", prefixedName) + } + target = f + } } else { - funcScope = obj.scope.Functions // func statement - } - - // Lookup function from scope... - f, exists := funcScope[obj.Name] - if !exists { - return fmt.Errorf("func `%s` does not exist in this scope", obj.Name) - } - - // Whether or not this is an ExprCall or ExprFunc, we do the same thing! - fn, isFn := f.(*ExprFunc) - if !isFn { - // this logic is now combined into the main execution flow... - //_, ok := f.(*ExprCall) - } - - if isFn && fn.Body != nil { - if i, j := len(obj.Args), len(fn.Args); i != j { - return fmt.Errorf("func `%s` is being called with %d args, but expected %d args", obj.Name, i, j) + // The call looks like f(). + prefixedName = obj.Name + f, exists := obj.scope.Functions[obj.Name] + if !exists { + return fmt.Errorf("func `%s` does not exist in this scope", prefixedName) } - } - // XXX: is this check or the above one logical here before unification? - if isFn && fn.Function != nil { - //if i, j := len(obj.Args), len(???.Args); i != j { - // return fmt.Errorf("func `%s` is being called with %d args, but expected %d args", obj.Name, i, j) - //} + target = f } - if isFn && len(fn.Values) > 0 { - // XXX: what can we add here? - } + if polymorphicTarget, isPolymorphic := target.(*ExprPoly); isPolymorphic { + // This function call refers to a polymorphic function expression. Those + // expressions can be instantiated at different types in different parts + // of the program, so the the definition we found has a "polymorphic" + // type. + // + // This particular call is one of the parts of the program which uses the + // polymorphic expression at a single, "monomorphic" type. We make a copy of + // the definition, and later each copy will be type-checked separately. + monomorphicTarget, err := polymorphicTarget.Definition.Copy() + if err != nil { + return errwrap.Wrapf(err, "could not copy the function definition `%s`", prefixedName) + } - // XXX: we do this twice, so we should avoid the first one somehow... - // XXX: why do we do it twice??? - if obj.expr != nil { - // possible programming error - //return fmt.Errorf("call already contains a func pointer") - } + // This call now has the only reference to monomorphicTarget, so it is + // our responsibility to scope-check it. We must use the scope which was + // captured at the definition site, not the scope argument we received + // as input, as that is the scope which is available at the call site. + definitionScope := polymorphicTarget.CapturedScope.Copy() - // FIXME: do we want scope or obj.fn.scope (below, and after it's set) ? - for i := len(scope.Chain) - 1; i >= 0; i-- { // reverse order - x, ok := scope.Chain[i].(*ExprCall) - if !ok { - continue + err = monomorphicTarget.SetScope(definitionScope, map[string]interfaces.Expr{}) + if err != nil { + return errwrap.Wrapf(err, "scope-checking the function definition `%s`", prefixedName) } - if x == obj.orig { // look for my original self - // scope chain found! - obj.expr = f // same pointer, don't copy - return fmt.Errorf("recursive func `%s` found", obj.Name) - //return nil // if recursion was supported + if obj.data.Debug { + obj.data.Logf("call $%s(): set scope: func pointer: %p (polymorphic) -> %p (copy)", prefixedName, &polymorphicTarget, &monomorphicTarget) } - } - - // Don't copy using interpolate, because we don't want to recursively - // copy things. We copy it for each use of the call. - // TODO: We want to recursively copy, but do we want to keep all the - // pointers the same, except for the obj.Args[i] ones that we stick in - // the scope for lookups...? - copied, err := f.Copy() // this does a light copy - if err != nil { - return errwrap.Wrapf(err, "could not copy expr") - } - obj.expr = copied - if obj.data.Debug { - obj.data.Logf("call(%s): set scope: func pointer: %p (before) -> %p (after)", obj.Name, f, copied) - } - - // Here, in the below loop, we want to do the equivalent of: - // `newScope.Variables["foo"] = obj.Args[i]`, which we can't because we - // only know the positional, indexed arguments. So, instead we build an - // indexed scope that is unpacked as such. - // Can't add the args `call:foo(42, "bar", true)` into the func scope... - //for i, arg := range obj.fn.Args { // copy - // newScope.Variables[arg.Name] = obj.Args[i] - //} - // Instead we use the special indexes to do that... - indexes := []interfaces.Expr{} - for _, arg := range obj.Args { - indexes = append(indexes, arg) - } - - // We start with the scope that the func had, and we augment it with our - // indexed arg variables, which will be needed in that scope. It is very - // important to *NOT* add the surrounding scope into the body because it - // shouldn't be able to jump into the function, only the args go into it - // from this point. We also need to extract the indexed args that are in - // the current scope that we've been building up via the SetScope stuff. - // FIXME: check I didn't pick the wrong scope in class/include... - s, err := getScope(obj.expr) - if err == ErrNoStoredScope { - s = interfaces.EmptyScope() - //s = scope // XXX: or this? - } else if err != nil { - // programming error? - return errwrap.Wrapf(err, "could not get scope from: %+v", obj.expr) - } - newScope := s.Copy() - //newScope := obj.fn.scope.Copy() // formerly - oldScope := scope.Copy() - - // We need to keep the function's scope, because that's what matters, - // but we need to augment it with the indexes we have currently. Plan: - // 1) Push indexes of "travelling" scope onto existing function scope. - // 2) Append to indexes any args that we're currently calling. - // 3) Propagate this new scope into the function. - // 4) In case of a future bug, consider dealing with this edge case! - if len(newScope.Indexes) > 0 { - // programming error ? - // TODO: this happens when we don't copy a static function... Is - // it a problem that we overwrite it below? It seems to be ok... - //return fmt.Errorf("edge case in ExprCall:SetScope, newScope is non-zero") - } - newScope.Indexes = oldScope.Indexes - newScope.PushIndexes(indexes) // obj.Args added to [0] - if obj.data.Debug { - obj.data.Logf("call(%s): set scope: adding to indexes: %+v", obj.Name, newScope.Indexes) + obj.expr = monomorphicTarget + } else { + // This call refers to a monomorphic expression which has already been + // scope-checked, so we don't need to scope-check it again. + obj.expr = target } - // recursion detection - newScope.Chain = append(newScope.Chain, obj.orig) // add expr to list - // TODO: switch based on obj.Var ? - //newScope.Functions[obj.Name] = copied // overwrite with new pointer - - err = obj.expr.SetScope(newScope) - return errwrap.Wrapf(err, "could not set call expr scope") + return nil } // SetType is used to set the type of this expression once it is known. This @@ -7839,14 +7786,24 @@ func (obj *ExprCall) Unify() ([]interfaces.Invariant, error) { } invariants = append(invariants, anyInvar) - // our type should equal the return type of the called function - callInvar := &interfaces.EqualityWrapCallInvariant{ - // TODO: should Expr1 and Expr2 be reversed??? - Expr1: obj, // return type expression from calling the function - Expr2Func: obj.expr, - // Expr2Args: obj.Args, XXX: ??? - } - invariants = append(invariants, callInvar) + // our type should equal the return type of the called function, and our + // argument types should be equal to the types of the parameters of the + // function + // arg0, arg1, arg2 + expr2Ord := []string{} + expr2Map := map[string]interfaces.Expr{} + for i, argExpr := range obj.Args { + argName := fmt.Sprintf("arg%d", i) + expr2Ord = append(expr2Ord, argName) + expr2Map[argName] = argExpr + } + funcInvar := &interfaces.EqualityWrapFuncInvariant{ + Expr1: obj.expr, + Expr2Map: expr2Map, + Expr2Ord: expr2Ord, + Expr2Out: obj, + } + invariants = append(invariants, funcInvar) // function specific code follows... fn, isFn := obj.expr.(*ExprFunc) @@ -8179,186 +8136,82 @@ func (obj *ExprCall) Unify() ([]interfaces.Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it, and // the edges from all of the child graphs to this. -func (obj *ExprCall) Graph() (*pgraph.Graph, error) { +func (obj *ExprCall) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { if obj.expr == nil { // possible programming error - return nil, fmt.Errorf("call doesn't contain an expr pointer yet") + return nil, nil, fmt.Errorf("call doesn't contain an expr pointer yet") } graph, err := pgraph.NewGraph("call") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - - // argnames! - argNames := []string{} - typ, err := obj.expr.Type() + ftyp, err := obj.expr.Type() if err != nil { - return nil, err - } - // TODO: can we use this method for all of the kinds of obj.expr? - // TODO: probably, but i've left in the expanded versions for now - argNames = typ.Ord - var inconsistentEdgeNames = false // probably better off with this off! - - // function specific code follows... - fn, isFn := obj.expr.(*ExprFunc) - if isFn && inconsistentEdgeNames { - if fn.Body != nil { - // add arg names that are seen in the ExprFunc struct! - a := []string{} - for _, x := range fn.Args { - a = append(a, x.Name) - } - argNames = a - } - if fn.Function != nil { - argNames = fn.function.Info().Sig.Ord - } - if len(fn.Values) > 0 { - // add the expected arg names from the selected function - typ, err := fn.Type() - if err != nil { - return nil, err - } - argNames = typ.Ord - } + return nil, nil, errwrap.Wrapf(err, "could not get the type of the function") } - if len(argNames) != len(obj.Args) { // extra safety... - return nil, fmt.Errorf("func `%s` expected %d args, got %d", obj.Name, len(argNames), len(obj.Args)) + // Add the vertex which produces the FuncValue. + exprGraph, funcValueFunc, err := obj.expr.Graph(env) + if err != nil { + return nil, nil, errwrap.Wrapf(err, "could not get the graph for the expr pointer") } + graph.AddGraph(exprGraph) + graph.AddVertex(funcValueFunc) - // Each func argument needs to point to the final function expression. - for pos, x := range obj.Args { // function arguments in order - g, err := x.Graph() + // Loop over the arguments, add them to the graph, but do _not_ connect them + // to the function vertex. Instead, each time the call vertex (which we + // create below) receives a FuncValue from the function node, it creates the + // corresponding subgraph and connects these arguments to it. + var argFuncs []interfaces.Func + for i, arg := range obj.Args { + argGraph, argFunc, err := arg.Graph(env) if err != nil { - return nil, err - } - - //argName := fmt.Sprintf("%d", pos) // indexed! - argName := argNames[pos] - edge := &interfaces.FuncEdge{Args: []string{argName}} - // TODO: replace with: - //edge := &interfaces.FuncEdge{Args: []string{fmt.Sprintf("arg:%s", argName)}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for func `%s`, arg `%s` was called twice", obj.Name, argName)) - } - once = true - return edge + return nil, nil, errwrap.Wrapf(err, "could not get graph for arg %d", i) } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // arg -> func + graph.AddGraph(argGraph) + argFuncs = append(argFuncs, argFunc) } - // This is important, because we don't want an extra, unnecessary edge! - if isFn && (fn.Function != nil || len(fn.Values) > 0) { - return graph, nil // built-in's don't need a vertex or an edge! + // Add a vertex for the call itself. + edgeName := structs.CallFuncArgNameFunction + callFunc := &structs.CallFunc{ + Type: obj.typ, + FuncType: ftyp, + EdgeName: edgeName, + ArgVertices: argFuncs, } + graph.AddVertex(callFunc) + graph.AddEdge(funcValueFunc, callFunc, &interfaces.FuncEdge{ + Args: []string{edgeName}, + }) - // Add the graph of the expression which must proceed the call... This - // might already exist in graph (i think)... - // Note: This can cause a panic if you get two NOT-connected vertices, - // in the source graph, because it tries to add two edges! Solution: add - // the missing edge between those in the source... Happy bug killing =D - graph.AddVertex(obj.expr) // duplicate additions are ignored and are harmless + return graph, callFunc, nil +} - g, err := obj.expr.Graph() - if err != nil { - return nil, err +// SetValue here is used to store the result of the last computation of this +// expression node after it has received all the required input values. This +// value is cached and can be retrieved by calling Value. +func (obj *ExprCall) SetValue(value types.Value) error { + if err := obj.typ.Cmp(value.Type()); err != nil { + return err } + obj.V = value + return nil +} - edge := &interfaces.FuncEdge{Args: []string{fmt.Sprintf("call:%s", obj.Name)}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for call `%s` was called twice", obj.Name)) - } - once = true - return edge +// Value returns the value of this expression in our type system. This will +// usually only be valid once the engine has run and values have been produced. +// This might get called speculatively (early) during unification to learn more. +// It is often unlikely that this kind of speculative execution finds something. +// This particular implementation of the function returns the previously stored +// and cached value as received by SetValue. +func (obj *ExprCall) Value() (types.Value, error) { + if obj.V == nil { + return nil, fmt.Errorf("func value does not yet exist") } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // expr -> call - - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -// Reminder that this looks very similar to ExprVar... -func (obj *ExprCall) Func() (interfaces.Func, error) { - if obj.expr == nil { - // possible programming error - return nil, fmt.Errorf("call doesn't contain an expr pointer yet") - } - - typ, err := obj.Type() - if err != nil { - return nil, err - } - - ftyp, err := obj.expr.Type() - if err != nil { - return nil, err - } - - // function specific code follows... - fn, isFn := obj.expr.(*ExprFunc) - if isFn && fn.Function != nil { - // NOTE: This has to be a unique pointer each time, which is why - // the ExprFunc builds a special unique copy into .function that - // is used here. If it was shared across the function graph, the - // function engine would error, because it would be operating on - // the same struct that is being touched from multiple places... - return fn.function, nil - //return obj.fn.Func() // this is incorrect. see ExprVar comment - } - - // XXX: receive the ExprFunc properly, and use it in CallFunc... - //if isFn && len(fn.Values) > 0 { - // return &structs.CallFunc{ - // Type: typ, // this is the type of what the func returns - // FuncType: ftyp, - // Edge: "???", - // Fn: ???, - // }, nil - //} - - // direct func - return &structs.CallFunc{ - Type: typ, // this is the type of what the func returns - FuncType: ftyp, - // the edge name used above in Graph is this... - Edge: fmt.Sprintf("call:%s", obj.Name), - //Indexed: true, // 0, 1, 2 ... TODO: is this useful? - }, nil -} - -// SetValue here is used to store the result of the last computation of this -// expression node after it has received all the required input values. This -// value is cached and can be retrieved by calling Value. -func (obj *ExprCall) SetValue(value types.Value) error { - if err := obj.typ.Cmp(value.Type()); err != nil { - return err - } - obj.V = value - return nil -} - -// Value returns the value of this expression in our type system. This will -// usually only be valid once the engine has run and values have been produced. -// This might get called speculatively (early) during unification to learn more. -// It is often unlikely that this kind of speculative execution finds something. -// This particular implementation of the function returns the previously stored -// and cached value as received by SetValue. -func (obj *ExprCall) Value() (types.Value, error) { - if obj.V == nil { - return nil, fmt.Errorf("func value does not yet exist") - } - return obj.V, nil + return obj.V, nil } // ExprVar is a representation of a variable lookup. It returns the expression @@ -8439,11 +8292,58 @@ func (obj *ExprVar) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph } // SetScope stores the scope for use in this resource. -func (obj *ExprVar) SetScope(scope *interfaces.Scope) error { - if scope == nil { - scope = interfaces.EmptyScope() +func (obj *ExprVar) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { + obj.scope = interfaces.EmptyScope() + if scope != nil { + obj.scope = scope.Copy() } - obj.scope = scope + + if monomorphicTarget, exists := context[obj.Name]; exists { + // This ExprVar refers to a parameter bound by an enclosing + // lambda definition. We do _not_ copy the definition, because + // it is already monomorphic. + obj.scope.Variables[obj.Name] = monomorphicTarget + + // There is no need to scope-check the target, it's just a + // an ExprParam with no internal references. + return nil + } + + target, exists := obj.scope.Variables[obj.Name] + if !exists { + return fmt.Errorf("variable %s not in scope", obj.Name) + } + + if polymorphicTarget, isPolymorphic := target.(*ExprPoly); isPolymorphic { + // This ExprVar refers to a polymorphic expression. Those + // expressions can be instantiated at different types in + // different parts of the program, so the definition we found + // has a "polymorphic" type. + // + // This particular ExprVar is one of the parts of the program + // which uses the polymorphic expression at a single, + // "monomorphic" type. We make a copy of the definition, and + // later each copy will be type-checked separately. + monomorphicTarget, err := polymorphicTarget.Definition.Copy() + if err != nil { + return errwrap.Wrapf(err, "copying the ExprPoly definition to which an ExprVar refers") + } + obj.scope.Variables[obj.Name] = monomorphicTarget + + // This ExprVar now has the only reference to monomorphicTarget, + // so it is our responsibility to scope-check it. We must use + // the scope which was captured at the definition site, not the + // scope argument we received as input, as that is the scope + // which is available at the use site. + definitionScope := polymorphicTarget.CapturedScope.Copy() + + return monomorphicTarget.SetScope(definitionScope, map[string]interfaces.Expr{}) + } + + // This ExprVar refers to a monomorphic expression which has already been + // scope-checked, so we don't need to scope-check it again. + obj.scope.Variables[obj.Name] = target + return nil } @@ -8500,8 +8400,6 @@ func (obj *ExprVar) Unify() ([]interfaces.Invariant, error) { invariants = append(invariants, invar) } - // don't recurse because we already got this through the bind statement - // FIXME: see the comment in StmtBind... keep this in for now... invars, err := expr.Unify() if err != nil { return nil, err @@ -8532,107 +8430,317 @@ func (obj *ExprVar) Unify() ([]interfaces.Invariant, error) { // important for filling the logical requirements of the graph type checker, and // to avoid duplicating production of the incoming input value from the bound // expression. -func (obj *ExprVar) Graph() (*pgraph.Graph, error) { - graph, err := pgraph.NewGraph("var") - if err != nil { - return nil, err +func (obj *ExprVar) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { + // Delegate to the target. + target := obj.scope.Variables[obj.Name] + graph, varFunc, err := target.Graph(env) + return graph, varFunc, err +} + +// SetValue here is a no-op, because algorithmically when this is called from +// the func engine, the child fields (the dest lookup expr) will have had this +// done to them first, and as such when we try and retrieve the set value from +// this expression by calling `Value`, it will build it from scratch! +func (obj *ExprVar) SetValue(value types.Value) error { + if err := obj.typ.Cmp(value.Type()); err != nil { + return err } - graph.AddVertex(obj) + // noop! + //obj.V = value + return nil +} - // ??? = $foo (this is the foo) - // lookup value from scope +// Value returns the value of this expression in our type system. This will +// usually only be valid once the engine has run and values have been produced. +// This might get called speculatively (early) during unification to learn more. +// This returns the value this variable points to. It is able to do so because +// it can lookup in the previous set scope which expression this points to, and +// then it can call Value on that expression. +func (obj *ExprVar) Value() (types.Value, error) { expr, exists := obj.scope.Variables[obj.Name] if !exists { - return nil, fmt.Errorf("var `%s` does not exist in this scope", obj.Name) + return nil, fmt.Errorf("var `%s` does not exist in scope", obj.Name) } + return expr.Value() // recurse +} - // should already exist in graph (i think)... - graph.AddVertex(expr) // duplicate additions are ignored and are harmless +// ExprParam represents a parameter to a function. +type ExprParam struct { + Name string // name of the parameter + Typ *types.Type +} + +// String returns a short representation of this expression. +func (obj *ExprParam) String() string { + return fmt.Sprintf("param(%s)", obj.Name) +} + +// Apply is a general purpose iterator method that operates on any AST node. It +// is not used as the primary AST traversal function because it is less readable +// and easy to reason about than manually implementing traversal for each node. +// Nevertheless, it is a useful facility for operations that might only apply to +// a select number of node types, since they won't need extra noop iterators... +func (obj *ExprParam) Apply(fn func(interfaces.Node) error) error { return fn(obj) } + +// Init initializes this branch of the AST, and returns an error if it fails to +// validate. +func (obj *ExprParam) Init(*interfaces.Data) error { + return langutil.ValidateVarName(obj.Name) +} + +// Interpolate returns a new node (aka a copy) once it has been expanded. This +// generally increases the size of the AST when it is used. It calls Interpolate +// on any child elements and builds the new node with those new node contents. +func (obj *ExprParam) Interpolate() (interfaces.Expr, error) { + return &ExprParam{ + Name: obj.Name, + Typ: obj.Typ, + }, nil +} + +// Copy returns a light copy of this struct. Anything static will not be copied. +// This intentionally returns a copy, because if a function (usually a lambda) +// that is used more than once, contains this variable, we will want each +// instantiation of it to be unique, otherwise they will be the same pointer, +// and they won't be able to have different values. +func (obj *ExprParam) Copy() (interfaces.Expr, error) { + return &ExprParam{ + Name: obj.Name, + Typ: obj.Typ, + }, nil +} - // the expr needs to point to the var lookup expression - g, err := expr.Graph() +// Ordering returns a graph of the scope ordering that represents the data flow. +// This can be used in SetScope so that it knows the correct order to run it in. +func (obj *ExprParam) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph, map[interfaces.Node]string, error) { + graph, err := pgraph.NewGraph("ordering") if err != nil { - return nil, err + return nil, nil, err + } + graph.AddVertex(obj) + + if obj.Name == "" { + return nil, nil, fmt.Errorf("missing param name") } + uid := paramOrderingPrefix + obj.Name // ordering id - edge := &interfaces.FuncEdge{Args: []string{fmt.Sprintf("var:%s", obj.Name)}} + cons := make(map[interfaces.Node]string) + cons[obj] = uid + + node, exists := produces[uid] + if exists { + edge := &pgraph.SimpleEdge{Name: "exprparam"} + graph.AddEdge(node, obj, edge) // prod -> cons + } - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for var `%s` was called twice", obj.Name)) + return graph, cons, nil +} + +// SetScope stores the scope for use in this resource. +func (obj *ExprParam) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { + // ExprParam doesn't have a scope, because it is the node to which a VarExpr + // can point to, it doesn't point to anything itself. + return nil +} + +// SetType is used to set the type of this expression once it is known. This +// usually happens during type unification, but it can also happen during +// parsing if a type is specified explicitly. Since types are static and don't +// change on expressions, if you attempt to set a different type than what has +// previously been set (when not initially known) this will error. +func (obj *ExprParam) SetType(typ *types.Type) error { + if obj.Typ != nil { + return obj.Typ.Cmp(typ) // if not set, ensure it doesn't change + } + obj.Typ = typ // set + return nil +} + +// Type returns the type of this expression. +func (obj *ExprParam) Type() (*types.Type, error) { + // Return the type if it is already known statically... It is useful for + // type unification to have some extra info early. + if obj.Typ == nil { + return nil, interfaces.ErrTypeCurrentlyUnknown + } + return obj.Typ, nil +} + +// Unify returns the list of invariants that this node produces. It recursively +// calls Unify on any children elements that exist in the AST, and returns the +// collection to the caller. +func (obj *ExprParam) Unify() ([]interfaces.Invariant, error) { + var invariants []interfaces.Invariant + + // if this was set explicitly by the parser + if obj.Typ != nil { + invar := &interfaces.EqualsInvariant{ + Expr: obj, + Type: obj.Typ, } - once = true - return edge + invariants = append(invariants, invar) } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // expr -> var - return graph, nil + return invariants, nil } -// Func returns a "pass-through" function which receives the bound value, and -// passes it to the consumer. This is essential for satisfying the type checker -// of the function graph engine. Reminder that this looks very similar to -// ExprCall... -func (obj *ExprVar) Func() (interfaces.Func, error) { - //expr, exists := obj.scope.Variables[obj.Name] - //if !exists { - // return nil, fmt.Errorf("var `%s` does not exist in scope", obj.Name) - //} +// Graph returns the reactive function graph which is expressed by this node. It +// includes any vertices produced by this node, and the appropriate edges to any +// vertices that are produced by its children. Nodes which fulfill the Expr +// interface directly produce vertices (and possible children) where as nodes +// that fulfill the Stmt interface do not produces vertices, where as their +// children might. +func (obj *ExprParam) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { + // Since ExprParam represents a function parameter, we want to receive values + // from the arguments passed to the function. The caller of ExprParam.Graph() + // should already know what those arguments are, so we can simply look up the + // argument by name in the environment. + paramFunc, exists := env[obj.Name] + if !exists { + return nil, nil, fmt.Errorf("param `%s` is not in the environment", obj.Name) + } - // this is wrong, if we did it this way, this expr wouldn't exist as a - // distinct node in the function graph to relay values through, instead, - // it would be acting as a "substitution/lookup" function, which just - // copies the bound function over into here. As a result, we'd have N - // copies of that function (based on the number of times N that that - // variable is used) instead of having that single bound function as - // input which is sent via N different edges to the multiple locations - // where the variables are used. Since the bound function would still - // have a single unique pointer, this wouldn't actually exist more than - // once in the graph, although since it's illogical, it causes the graph - // type checking (the edge counting in the function graph engine) to - // notice a problem and error. - //return expr.Func() // recurse? - - // instead, return a function which correctly does a lookup in the scope - // and returns *that* stream of values instead. - typ, err := obj.Type() + graph, err := pgraph.NewGraph("ExprParam") + if err != nil { + return nil, nil, err + } + graph.AddVertex(paramFunc) + return graph, paramFunc, nil +} + + +// SetValue here is a no-op, because algorithmically when this is called from +// the func engine, the child fields (the dest lookup expr) will have had this +// done to them first, and as such when we try and retrieve the set value from +// this expression by calling `Value`, it will build it from scratch! +func (obj *ExprParam) SetValue(value types.Value) error { + // ignored, as we don't support ExprParam.Value() + return nil +} + +// Value returns the value of this expression in our type system. This will +// usually only be valid once the engine has run and values have been produced. +// This might get called speculatively (early) during unification to learn more. +func (obj *ExprParam) Value() (types.Value, error) { + return nil, nil +} + +// ExprPoly is a polymorphic expression that is a definition that can be used in +// multiple places with different types. We must copy the definition at each +// call site in order for the type checker to find a different type at each call +// site. We create this copy inside SetScope, at which point we also recursively +// call SetScope on the copy. We must be careful to use the scope captured at +// the definition site, not the scope which is available at the call site. +type ExprPoly struct { + Definition interfaces.Expr // The definition. + CapturedScope *interfaces.Scope // The scope at the definition site. +} + +// String returns a short representation of this expression. +func (obj *ExprPoly) String() string { + return fmt.Sprintf("polymorphic(%s)", obj.Definition.String()) +} + +// Apply is a general purpose iterator method that operates on any AST node. It +// is not used as the primary AST traversal function because it is less readable +// and easy to reason about than manually implementing traversal for each node. +// Nevertheless, it is a useful facility for operations that might only apply to +// a select number of node types, since they won't need extra noop iterators... +func (obj *ExprPoly) Apply(fn func(interfaces.Node) error) error { + if err := obj.Definition.Apply(fn); err != nil { + return err + } + return fn(obj) +} + +// Init initializes this branch of the AST, and returns an error if it fails to +// validate. +func (obj *ExprPoly) Init(data *interfaces.Data) error { + return obj.Definition.Init(data) +} + +// Interpolate returns a new node (aka a copy) once it has been expanded. This +// generally increases the size of the AST when it is used. It calls Interpolate +// on any child elements and builds the new node with those new node contents. +func (obj *ExprPoly) Interpolate() (interfaces.Expr, error) { + definition, err := obj.Definition.Interpolate() if err != nil { return nil, err } - // var func - return &structs.VarFunc{ - Type: typ, - Edge: fmt.Sprintf("var:%s", obj.Name), // the edge name used above in Graph is this... + return &ExprPoly{ + Definition: definition, + CapturedScope: obj.CapturedScope, }, nil } +// Copy returns a light copy of this struct. Anything static will not be copied. +// This implementation intentionally does not copy anything, because the +// Definition is already intended to be copied at each use site. +func (obj *ExprPoly) Copy() (interfaces.Expr, error) { + return obj, nil +} + +// Ordering returns a graph of the scope ordering that represents the data flow. +// This can be used in SetScope so that it knows the correct order to run it in. +func (obj *ExprPoly) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph, map[interfaces.Node]string, error) { + return obj.Definition.Ordering(produces) +} + +// SetScope stores the scope for use in this resource. +func (obj *ExprPoly) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { + // Don't recur into the definition yet; instead, capture the scope for later, + // so that ExprVar can call SetScope on the definition at each use site. + obj.CapturedScope = scope + return nil +} + +// SetType is used to set the type of this expression once it is known. This +// usually happens during type unification, but it can also happen during +// parsing if a type is specified explicitly. Since types are static and don't +// change on expressions, if you attempt to set a different type than what has +// previously been set (when not initially known) this will error. +func (obj *ExprPoly) SetType(typ *types.Type) error { + panic("ExprPoly.SetType(): should not happen, all ExprPoly expressions should be gone by the time type-checking starts") +} + +// Type returns the type of this expression. +func (obj *ExprPoly) Type() (*types.Type, error) { + return nil, interfaces.ErrTypeCurrentlyUnknown +} + +// Unify returns the list of invariants that this node produces. It recursively +// calls Unify on any children elements that exist in the AST, and returns the +// collection to the caller. +func (obj *ExprPoly) Unify() ([]interfaces.Invariant, error) { + panic("ExprPoly.Unify(): should not happen, all ExprPoly expressions should be gone by the time type-checking starts") +} + +// Graph returns the reactive function graph which is expressed by this node. It +// includes any vertices produced by this node, and the appropriate edges to any +// vertices that are produced by its children. Nodes which fulfill the Expr +// interface directly produce vertices (and possible children) where as nodes +// that fulfill the Stmt interface do not produces vertices, where as their +// children might. +func (obj *ExprPoly) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { + panic("ExprPoly.Unify(): should not happen, all ExprPoly expressions should be gone by the time type-checking starts") +} + + // SetValue here is a no-op, because algorithmically when this is called from // the func engine, the child fields (the dest lookup expr) will have had this // done to them first, and as such when we try and retrieve the set value from // this expression by calling `Value`, it will build it from scratch! -func (obj *ExprVar) SetValue(value types.Value) error { - if err := obj.typ.Cmp(value.Type()); err != nil { - return err - } - // noop! - //obj.V = value +func (obj *ExprPoly) SetValue(value types.Value) error { + // ignored, as we don't support ExprPoly.Value() return nil } // Value returns the value of this expression in our type system. This will // usually only be valid once the engine has run and values have been produced. // This might get called speculatively (early) during unification to learn more. -// This returns the value this variable points to. It is able to do so because -// it can lookup in the previous set scope which expression this points to, and -// then it can call Value on that expression. -func (obj *ExprVar) Value() (types.Value, error) { - expr, exists := obj.scope.Variables[obj.Name] - if !exists { - return nil, fmt.Errorf("var `%s` does not exist in scope", obj.Name) - } - return expr.Value() // recurse +func (obj *ExprPoly) Value() (types.Value, error) { + return nil, nil } // ExprIf represents an if expression which *must* have both branches, and which @@ -8828,18 +8936,18 @@ func (obj *ExprIf) Ordering(produces map[string]interfaces.Node) (*pgraph.Graph, // SetScope stores the scope for later use in this resource and its children, // which it propagates this downwards to. -func (obj *ExprIf) SetScope(scope *interfaces.Scope) error { +func (obj *ExprIf) SetScope(scope *interfaces.Scope, context map[string]interfaces.Expr) error { if scope == nil { scope = interfaces.EmptyScope() } obj.scope = scope - if err := obj.ThenBranch.SetScope(scope); err != nil { + if err := obj.ThenBranch.SetScope(scope, context); err != nil { return err } - if err := obj.ElseBranch.SetScope(scope); err != nil { + if err := obj.ElseBranch.SetScope(scope, context); err != nil { return err } - return obj.Condition.SetScope(scope) + return obj.Condition.SetScope(scope, context) } // SetType is used to set the type of this expression once it is known. This @@ -8858,7 +8966,7 @@ func (obj *ExprIf) SetType(typ *types.Type) error { // Type returns the type of this expression. func (obj *ExprIf) Type() (*types.Type, error) { boolValue, err := obj.Condition.Value() // attempt early speculation - if err == nil && obj.typ == nil { + if err == nil && obj.typ == nil && boolValue != nil { branch := obj.ElseBranch if boolValue.Bool() { // must not panic branch = obj.ThenBranch @@ -8939,6 +9047,19 @@ func (obj *ExprIf) Unify() ([]interfaces.Invariant, error) { return invariants, nil } +// Func returns a function which returns the correct branch based on the ever +// changing conditional boolean input. +func (obj *ExprIf) Func() (interfaces.Func, error) { + typ, err := obj.Type() + if err != nil { + return nil, err + } + + return &structs.IfFunc{ + Type: typ, // this is the output type of the expression + }, nil +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr @@ -8949,12 +9070,15 @@ func (obj *ExprIf) Unify() ([]interfaces.Invariant, error) { // shouldn't have any ill effects. // XXX: is this completely true if we're running technically impure, but safe // built-in functions on both branches? Can we turn off half of this? -func (obj *ExprIf) Graph() (*pgraph.Graph, error) { +func (obj *ExprIf) Graph(env map[string]interfaces.Func) (*pgraph.Graph, interfaces.Func, error) { graph, err := pgraph.NewGraph("if") if err != nil { - return nil, err + return nil, nil, err + } + function, err := obj.Func() + if err != nil { + return nil, nil, err } - graph.AddVertex(obj) exprs := map[string]interfaces.Expr{ "c": obj.Condition, @@ -8963,38 +9087,17 @@ func (obj *ExprIf) Graph() (*pgraph.Graph, error) { } for _, argName := range []string{"c", "a", "b"} { // deterministic order x := exprs[argName] - g, err := x.Graph() + g, f, err := x.Graph(env) if err != nil { - return nil, err + return nil, nil, err } + graph.AddGraph(g) edge := &interfaces.FuncEdge{Args: []string{argName}} - - var once bool - edgeGenFn := func(v1, v2 pgraph.Vertex) pgraph.Edge { - if once { - panic(fmt.Sprintf("edgeGenFn for ifexpr edge `%s` was called twice", argName)) - } - once = true - return edge - } - graph.AddEdgeGraphVertexLight(g, obj, edgeGenFn) // branch -> if + graph.AddEdge(f, function, edge) // branch -> if } - return graph, nil -} - -// Func returns a function which returns the correct branch based on the ever -// changing conditional boolean input. -func (obj *ExprIf) Func() (interfaces.Func, error) { - typ, err := obj.Type() - if err != nil { - return nil, err - } - - return &structs.IfFunc{ - Type: typ, // this is the output type of the expression - }, nil + return graph, function, nil } // SetValue here is a no-op, because algorithmically when this is called from diff --git a/lang/ast/util.go b/lang/ast/util.go index 1918cd6885..d0b9806169 100644 --- a/lang/ast/util.go +++ b/lang/ast/util.go @@ -70,7 +70,19 @@ func FuncPrefixToFunctionsScope(prefix string) map[string]interfaces.Expr { } exprs[name] = fn } - return exprs + + // Wrap every Expr in ExprPoly, so that the function can be used with + // different types. Those functions are all builtins, so they don't need to + // access the surrounding scope. + exprPolys := make(map[string]interfaces.Expr) + for name, expr := range exprs { + exprPolys[name] = &ExprPoly{ + Definition: expr, + CapturedScope: interfaces.EmptyScope(), + } + } + + return exprPolys } // VarPrefixToVariablesScope is a helper function to return the variables @@ -196,16 +208,6 @@ func ValueToExpr(val types.Value) (interfaces.Expr, error) { Fields: fields, } - case *types.FuncValue: - // TODO: this particular case is particularly untested! - expr = &ExprFunc{ - Title: "", // TODO: change this? - // TODO: symmetrically, it would have used x.Func() here - Values: []*types.FuncValue{ - x, // just one! - }, - } - case *types.VariantValue: // TODO: should this be allowed, or should we unwrap them? return nil, fmt.Errorf("variant values are not supported") diff --git a/lang/funcs/contains_func.go b/lang/funcs/contains_func.go index e0b21d39a0..954357b1db 100644 --- a/lang/funcs/contains_func.go +++ b/lang/funcs/contains_func.go @@ -41,6 +41,8 @@ func init() { Register(ContainsFuncName, func() interfaces.Func { return &ContainsFunc{} }) // must register the func and name } +var _ interfaces.PolyFunc = &ContainsFunc{} // ensure it meets this expectation + // ContainsFunc returns true if a value is found in a list. Otherwise false. type ContainsFunc struct { Type *types.Type // this is the type of value stored in our list @@ -291,46 +293,46 @@ func (obj *ContainsFunc) Polymorphisms(partialType *types.Type, partialValues [] // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *ContainsFunc) Build(typ *types.Type) error { +func (obj *ContainsFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 { - return fmt.Errorf("the contains function needs exactly two args") + return nil, fmt.Errorf("the contains function needs exactly two args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } tNeedle, exists := typ.Map[typ.Ord[0]] if !exists || tNeedle == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } tHaystack, exists := typ.Map[typ.Ord[1]] if !exists || tHaystack == nil { - return fmt.Errorf("second arg must be specified") + return nil, fmt.Errorf("second arg must be specified") } if tHaystack.Kind != types.KindList { - return fmt.Errorf("second argument must be of kind list") + return nil, fmt.Errorf("second argument must be of kind list") } if err := tHaystack.Val.Cmp(tNeedle); err != nil { - return errwrap.Wrapf(err, "type of first arg must match type of list elements in second arg") + return nil, errwrap.Wrapf(err, "type of first arg must match type of list elements in second arg") } if err := typ.Out.Cmp(types.TypeBool); err != nil { - return errwrap.Wrapf(err, "return type must be a boolean") + return nil, errwrap.Wrapf(err, "return type must be a boolean") } obj.Type = tNeedle // type of value stored in our list - return nil + return obj.sig(), nil } // Validate tells us if the input struct takes a valid form. @@ -346,8 +348,7 @@ func (obj *ContainsFunc) Validate() error { func (obj *ContainsFunc) Info() *interfaces.Info { var sig *types.Type if obj.Type != nil { // don't panic if called speculatively - s := obj.Type.String() - sig = types.NewType(fmt.Sprintf("func(%s %s, %s []%s) bool", containsArgNameNeedle, s, containsArgNameHaystack, s)) + sig = obj.sig() // helper } return &interfaces.Info{ Pure: true, @@ -357,6 +358,12 @@ func (obj *ContainsFunc) Info() *interfaces.Info { } } +// helper +func (obj *ContainsFunc) sig() *types.Type { + s := obj.Type.String() + return types.NewType(fmt.Sprintf("func(%s %s, %s []%s) bool", containsArgNameNeedle, s, containsArgNameHaystack, s)) +} + // Init runs some startup code for this function. func (obj *ContainsFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/core/fmt/printf_func.go b/lang/funcs/core/fmt/printf_func.go index 2796528753..2ca271501a 100644 --- a/lang/funcs/core/fmt/printf_func.go +++ b/lang/funcs/core/fmt/printf_func.go @@ -53,6 +53,8 @@ func init() { funcs.ModuleRegister(ModuleName, PrintfFuncName, func() interfaces.Func { return &PrintfFunc{} }) } +var _ interfaces.PolyFunc = &PrintfFunc{} // ensure it meets this expectation + // PrintfFunc is a static polymorphic function that compiles a format string and // returns the output as a string. It bases its output on the values passed in // to it. It examines the type of the arguments at compile time and then @@ -437,33 +439,49 @@ func (obj *PrintfFunc) Polymorphisms(partialType *types.Type, partialValues []ty // Build takes the now known function signature and stores it so that this // function can appear to be static. That type is used to build our function // statically. -func (obj *PrintfFunc) Build(typ *types.Type) error { +func (obj *PrintfFunc) Build(typ *types.Type) (*types.Type, error) { if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) < 1 { - return fmt.Errorf("the printf function needs at least one arg") + return nil, fmt.Errorf("the printf function needs at least one arg") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Out.Cmp(types.TypeStr) != nil { - return fmt.Errorf("return type of function must be an str") + return nil, fmt.Errorf("return type of function must be an str") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } t0, exists := typ.Map[typ.Ord[0]] if !exists || t0 == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } if t0.Cmp(types.TypeStr) != nil { - return fmt.Errorf("first arg for printf must be an str") + return nil, fmt.Errorf("first arg for printf must be an str") } - obj.Type = typ // function type is now known! - return nil + //newTyp := typ.Copy() + newTyp := &types.Type{ + Kind: typ.Kind, // copy + Map: make(map[string]*types.Type), // new + Ord: []string{}, // new + Out: typ.Out, // copy + } + for i, x := range typ.Ord { // remap arg names + argName, err := obj.ArgGen(i) + if err != nil { + return nil, err + } + newTyp.Map[argName] = typ.Map[x] + newTyp.Ord = append(newTyp.Ord, argName) + } + + obj.Type = newTyp // function type is now known! + return obj.Type, nil } // Validate makes sure we've built our struct properly. It is usually unused for diff --git a/lang/funcs/core/iter/map_func.go b/lang/funcs/core/iter/map_func.go index c09eb818a1..b0023aaadc 100644 --- a/lang/funcs/core/iter/map_func.go +++ b/lang/funcs/core/iter/map_func.go @@ -22,8 +22,10 @@ import ( "fmt" "github.com/purpleidea/mgmt/lang/funcs" + "github.com/purpleidea/mgmt/lang/funcs/structs" "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/lang/types/full" "github.com/purpleidea/mgmt/util" "github.com/purpleidea/mgmt/util/errwrap" ) @@ -31,16 +33,17 @@ import ( const ( // MapFuncName is the name this function is registered as. MapFuncName = "map" + + // arg names... + mapArgNameInputs = "inputs" + mapArgNameFunction = "function" ) func init() { funcs.ModuleRegister(ModuleName, MapFuncName, func() interfaces.Func { return &MapFunc{} }) // must register the func and name } -const ( - argNameInputs = "inputs" - argNameFunction = "function" -) +var _ interfaces.PolyFunc = &MapFunc{} // ensure it meets this expectation // MapFunc is the standard map iterator function that applies a function to each // element in a list. It returns a list with the same number of elements as the @@ -59,10 +62,16 @@ type MapFunc struct { init *interfaces.Init last types.Value // last value received to use for diff - inputs types.Value - function func([]types.Value) (types.Value, error) + lastFuncValue *full.FuncValue // remember the last function value + lastInputListLength int // remember the last input list length + + inputListType *types.Type + outputListType *types.Type - result types.Value // last calculated output + // outputChan is an initially-nil channel from which we receive output + // lists from the subgraph. This channel is reset when the subgraph is + // recreated. + outputChan chan types.Value } // String returns a simple name for this function. This is needed so this struct @@ -73,7 +82,7 @@ func (obj *MapFunc) String() string { // ArgGen returns the Nth arg name for this function. func (obj *MapFunc) ArgGen(index int) (string, error) { - seq := []string{argNameInputs, argNameFunction} // inverted for pretty! + seq := []string{mapArgNameInputs, mapArgNameFunction} // inverted for pretty! if l := len(seq); index >= l { return "", fmt.Errorf("index %d exceeds arg length of %d", index, l) } @@ -439,7 +448,7 @@ func (obj *MapFunc) Polymorphisms(partialType *types.Type, partialValues []types tI := types.NewType(fmt.Sprintf("[]%s", t1.String())) // in tO := types.NewType(fmt.Sprintf("[]%s", t2.String())) // out tF := types.NewType(fmt.Sprintf("func(%s) %s", t1.String(), t2.String())) - s := fmt.Sprintf("func(%s %s, %s %s) %s", argNameInputs, tI, argNameFunction, tF, tO) + s := fmt.Sprintf("func(%s %s, %s %s) %s", mapArgNameInputs, tI, mapArgNameFunction, tF, tO) typ := types.NewType(s) // yay! // TODO: type check that the partialValues are compatible @@ -452,66 +461,67 @@ func (obj *MapFunc) Polymorphisms(partialType *types.Type, partialValues []types // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *MapFunc) Build(typ *types.Type) error { +func (obj *MapFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 { - return fmt.Errorf("the map needs exactly two args") + return nil, fmt.Errorf("the map needs exactly two args") } if typ.Map == nil { - return fmt.Errorf("the map is nil") + return nil, fmt.Errorf("the map is nil") } tInputs, exists := typ.Map[typ.Ord[0]] if !exists || tInputs == nil { - return fmt.Errorf("first argument was missing") + return nil, fmt.Errorf("first argument was missing") } tFunction, exists := typ.Map[typ.Ord[1]] if !exists || tFunction == nil { - return fmt.Errorf("second argument was missing") + return nil, fmt.Errorf("second argument was missing") } if tInputs.Kind != types.KindList { - return fmt.Errorf("first argument must be of kind list") + return nil, fmt.Errorf("first argument must be of kind list") } if tFunction.Kind != types.KindFunc { - return fmt.Errorf("second argument must be of kind func") + return nil, fmt.Errorf("second argument must be of kind func") } if typ.Out == nil { - return fmt.Errorf("return type must be specified") + return nil, fmt.Errorf("return type must be specified") } if typ.Out.Kind != types.KindList { - return fmt.Errorf("return argument must be a list") + return nil, fmt.Errorf("return argument must be a list") } if len(tFunction.Ord) != 1 { - return fmt.Errorf("the functions map needs exactly one arg") + return nil, fmt.Errorf("the functions map needs exactly one arg") } if tFunction.Map == nil { - return fmt.Errorf("the functions map is nil") + return nil, fmt.Errorf("the functions map is nil") } tArg, exists := tFunction.Map[tFunction.Ord[0]] if !exists || tArg == nil { - return fmt.Errorf("the functions first argument was missing") + return nil, fmt.Errorf("the functions first argument was missing") } if err := tArg.Cmp(tInputs.Val); err != nil { - return errwrap.Wrapf(err, "the functions arg type must match the input list contents type") + return nil, errwrap.Wrapf(err, "the functions arg type must match the input list contents type") } if tFunction.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if err := tFunction.Out.Cmp(typ.Out.Val); err != nil { - return errwrap.Wrapf(err, "return type of function must match returned list contents type") + return nil, errwrap.Wrapf(err, "return type of function must match returned list contents type") } obj.Type = tInputs.Val // or tArg obj.RType = tFunction.Out // or typ.Out.Val - return nil + + return obj.sig(), nil } // Validate tells us if the input struct takes a valid form. @@ -525,6 +535,18 @@ func (obj *MapFunc) Validate() error { // Info returns some static info about itself. Build must be called before this // will return correct data. func (obj *MapFunc) Info() *interfaces.Info { + sig := obj.sig() // helper + + return &interfaces.Info{ + Pure: false, // TODO: what if the input function isn't pure? + Memo: false, + Sig: sig, + Err: obj.Validate(), + } +} + +// helper +func (obj *MapFunc) sig() *types.Type { // TODO: what do we put if this is unknown? tIi := types.TypeVariant if obj.Type != nil { @@ -539,87 +561,252 @@ func (obj *MapFunc) Info() *interfaces.Info { tO := types.NewType(fmt.Sprintf("[]%s", tOi.String())) // return type // type of 1st arg (the function) - tF := types.NewType(fmt.Sprintf("func(%s) %s", tIi.String(), tOi.String())) + tF := types.NewType(fmt.Sprintf("func(%s %s) %s", "name-which-can-vary-over-time", tIi.String(), tOi.String())) - s := fmt.Sprintf("func(%s %s, %s %s) %s", argNameInputs, tI, argNameFunction, tF, tO) - typ := types.NewType(s) // yay! - - return &interfaces.Info{ - Pure: false, // TODO: what if the input function isn't pure? - Memo: false, - Sig: typ, - Err: obj.Validate(), - } + s := fmt.Sprintf("func(%s %s, %s %s) %s", mapArgNameInputs, tI, mapArgNameFunction, tF, tO) + return types.NewType(s) // yay! } // Init runs some startup code for this function. func (obj *MapFunc) Init(init *interfaces.Init) error { obj.init = init + obj.lastFuncValue = nil + obj.lastInputListLength = -1 + + obj.inputListType = types.NewType(fmt.Sprintf("[]%s", obj.Type)) + obj.outputListType = types.NewType(fmt.Sprintf("[]%s", obj.RType)) + return nil } // Stream returns the changing values that this func has over time. func (obj *MapFunc) Stream(ctx context.Context) error { + // Every time the FuncValue or the length of the list changes, recreate the + // subgraph, by calling the FuncValue N times on N nodes, each of which + // extracts one of the N values in the list. + defer close(obj.init.Output) // the sender closes - rtyp := types.NewType(fmt.Sprintf("[]%s", obj.RType.String())) + + // A Func to send input lists to the subgraph. The Txn.Erase() call ensures + // that this Func is not removed when the subgraph is recreated, so that the + // function graph can propagate the last list we received to the subgraph. + inputChan := make(chan types.Value) + subgraphInput := &structs.ChannelBasedSourceFunc{ + Name: "subgraphInput", + Source: obj, + Chan: inputChan, + Type: obj.inputListType, + } + obj.init.Txn.AddVertex(subgraphInput) + if err := obj.init.Txn.Commit(); err != nil { + return errwrap.Wrapf(err, "commit error in Stream") + } + obj.init.Txn.Erase() // prevent the next Reverse() from removing subgraphInput + defer func() { + close(inputChan) + obj.init.Txn.Reverse() + obj.init.Txn.DeleteVertex(subgraphInput) + obj.init.Txn.Commit() + }() + + obj.outputChan = nil + + canReceiveMoreFuncValuesOrInputLists := true + canReceiveMoreOutputLists := true for { + + if !canReceiveMoreFuncValuesOrInputLists && !canReceiveMoreOutputLists { + //break + return nil + } + select { case input, ok := <-obj.init.Input: if !ok { - obj.init.Input = nil // don't infinite loop back - continue // no more inputs, but don't return! + obj.init.Input = nil // block looping back here + canReceiveMoreFuncValuesOrInputLists = false + continue } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} + // XXX: double check this passes through function changes if obj.last != nil && input.Cmp(obj.last) == nil { continue // value didn't change, skip it } obj.last = input // store for next - function := input.Struct()[argNameFunction].Func() // func([]Value) (Value, error) - //if function == obj.function { // TODO: how can we cmp? - // continue // nothing changed - //} - obj.function = function - - inputs := input.Struct()[argNameInputs] - if obj.inputs != nil && obj.inputs.Cmp(inputs) == nil { - continue // nothing changed - } - obj.inputs = inputs - - // run the function on each index - output := []types.Value{} - for ix, v := range inputs.List() { // []Value - args := []types.Value{v} // only one input arg! - x, err := function(args) - if err != nil { - return errwrap.Wrapf(err, "error running map function on index %d", ix) + value, exists := input.Struct()[mapArgNameFunction] + if !exists { + return fmt.Errorf("programming error, can't find edge") + } + + newFuncValue, ok := value.(*full.FuncValue) + if !ok { + return fmt.Errorf("programming error, can't convert to *FuncValue") + } + + newInputList, exists := input.Struct()[mapArgNameInputs] + if !exists { + return fmt.Errorf("programming error, can't find edge") + } + + // If we have a new function or the length of the input + // list has changed, then we need to replace the + // subgraph with a new one that uses the new function + // the correct number of times. + + // It's important to have this compare step to avoid + // redundant graph replacements which slow things down, + // but also cause the engine to lock, which can preempt + // the process scheduler, which can cause duplicate or + // unnecessary re-sending of values here, which causes + // the whole process to repeat ad-nauseum. + n := len(newInputList.List()) + if newFuncValue != obj.lastFuncValue || n != obj.lastInputListLength { + obj.lastFuncValue = newFuncValue + obj.lastInputListLength = n + // replaceSubGraph uses the above two values + if err := obj.replaceSubGraph(subgraphInput); err != nil { + return errwrap.Wrapf(err, "could not replace subgraph") } + canReceiveMoreOutputLists = true + } - output = append(output, x) + // send the new input list to the subgraph + select { + case inputChan <- newInputList: + case <-ctx.Done(): + return nil } - result := &types.ListValue{ - V: output, - T: rtyp, + + case outputList, ok := <-obj.outputChan: + // send the new output list downstream + if !ok { + obj.outputChan = nil + canReceiveMoreOutputLists = false + continue } - if obj.result != nil && obj.result.Cmp(result) == nil { - continue // result didn't change + select { + case obj.init.Output <- outputList: + case <-ctx.Done(): + return nil } - obj.result = result // store new result case <-ctx.Done(): return nil } + } +} - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil +func (obj *MapFunc) replaceSubGraph(subgraphInput interfaces.Func) error { + // Create a subgraph which splits the input list into 'n' nodes, applies + // 'newFuncValue' to each, then combines the 'n' outputs back into a list. + // + // Here is what the subgraph looks like: + // + // digraph { + // "subgraphInput" -> "inputElemFunc0" + // "subgraphInput" -> "inputElemFunc1" + // "subgraphInput" -> "inputElemFunc2" + // + // "inputElemFunc0" -> "outputElemFunc0" + // "inputElemFunc1" -> "outputElemFunc1" + // "inputElemFunc2" -> "outputElemFunc2" + // + // "outputElemFunc0" -> "outputListFunc" + // "outputElemFunc1" -> "outputListFunc" + // "outputElemFunc1" -> "outputListFunc" + // + // "outputListFunc" -> "subgraphOutput" + // } + + const channelBasedSinkFuncArgNameEdgeName = structs.ChannelBasedSinkFuncArgName // XXX: not sure if the specific name matters. + + // delete the old subgraph + if err := obj.init.Txn.Reverse(); err != nil { + return errwrap.Wrapf(err, "could not Reverse") + } + + // create the new subgraph + + obj.outputChan = make(chan types.Value) + subgraphOutput := &structs.ChannelBasedSinkFunc{ + Name: "subgraphOutput", + Target: obj, + EdgeName: channelBasedSinkFuncArgNameEdgeName, + Chan: obj.outputChan, + Type: obj.outputListType, + } + obj.init.Txn.AddVertex(subgraphOutput) + + m := make(map[string]*types.Type) + ord := []string{} + for i := 0; i < obj.lastInputListLength; i++ { + argName := fmt.Sprintf("outputElem%d", i) + m[argName] = obj.RType + ord = append(ord, argName) + } + typ := &types.Type{ + Kind: types.KindFunc, + Map: m, + Ord: ord, + Out: obj.outputListType, + } + outputListFunc := structs.SimpleFnToDirectFunc( + "mapOutputList", + &types.FuncValue{ + V: func(args []types.Value) (types.Value, error) { + listValue := &types.ListValue{ + V: args, + T: obj.outputListType, + } + + return listValue, nil + }, + T: typ, + }, + ) + + obj.init.Txn.AddVertex(outputListFunc) + obj.init.Txn.AddEdge(outputListFunc, subgraphOutput, &interfaces.FuncEdge{ + Args: []string{channelBasedSinkFuncArgNameEdgeName}, + }) + + for i := 0; i < obj.lastInputListLength; i++ { + i := i + inputElemFunc := structs.SimpleFnToDirectFunc( + fmt.Sprintf("mapInputElem[%d]", i), + &types.FuncValue{ + V: func(args []types.Value) (types.Value, error) { + if len(args) != 1 { + return nil, fmt.Errorf("inputElemFunc: expected a single argument") + } + arg := args[0] + + list, ok := arg.(*types.ListValue) + if !ok { + return nil, fmt.Errorf("inputElemFunc: expected a ListValue argument") + } + + return list.List()[i], nil + }, + T: types.NewType(fmt.Sprintf("func(inputList %s) %s", obj.inputListType, obj.Type)), + }, + ) + obj.init.Txn.AddVertex(inputElemFunc) + + outputElemFunc, err := obj.lastFuncValue.Call(obj.init.Txn, []interfaces.Func{inputElemFunc}) + if err != nil { + return errwrap.Wrapf(err, "could not call obj.lastFuncValue.Call()") } + + obj.init.Txn.AddEdge(subgraphInput, inputElemFunc, &interfaces.FuncEdge{ + Args: []string{"inputList"}, + }) + obj.init.Txn.AddEdge(outputElemFunc, outputListFunc, &interfaces.FuncEdge{ + Args: []string{fmt.Sprintf("outputElem%d", i)}, + }) } + + return obj.init.Txn.Commit() } diff --git a/lang/funcs/core/template_func.go b/lang/funcs/core/template_func.go index 75b05bef55..5db524822e 100644 --- a/lang/funcs/core/template_func.go +++ b/lang/funcs/core/template_func.go @@ -55,6 +55,8 @@ func init() { funcs.Register(TemplateFuncName, func() interfaces.Func { return &TemplateFunc{} }) } +var _ interfaces.PolyFunc = &TemplateFunc{} // ensure it meets this expectation + // TemplateFunc is a static polymorphic function that compiles a template and // returns the output as a string. It bases its output on the values passed in // to it. It examines the type of the second argument (the input data vars) at @@ -66,9 +68,8 @@ type TemplateFunc struct { // Type is the type of the input vars (2nd) arg if one is specified. Nil // is the special undetermined value that is used before type is known. Type *types.Type // type of vars - // NoVars is set to true instead of specifying Type if we have a boring - // template that takes no args. - NoVars bool + + built bool // was this function built yet? init *interfaces.Init last types.Value // last value received to use for diff @@ -296,50 +297,51 @@ func (obj *TemplateFunc) Polymorphisms(partialType *types.Type, partialValues [] // function can appear to be static. It extracts the type of the vars argument, // which is the dynamic part which can change. That type is used to build our // function statically. -func (obj *TemplateFunc) Build(typ *types.Type) error { +func (obj *TemplateFunc) Build(typ *types.Type) (*types.Type, error) { if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 && len(typ.Ord) != 1 { - return fmt.Errorf("the template function needs exactly one or two args") + return nil, fmt.Errorf("the template function needs exactly one or two args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Out.Cmp(types.TypeStr) != nil { - return fmt.Errorf("return type of function must be an str") + return nil, fmt.Errorf("return type of function must be an str") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } t0, exists := typ.Map[typ.Ord[0]] if !exists || t0 == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } if t0.Cmp(types.TypeStr) != nil { - return fmt.Errorf("first arg for template must be an str") + return nil, fmt.Errorf("first arg for template must be an str") } if len(typ.Ord) == 1 { // no args being passed in (boring template) - obj.NoVars = true - return nil + obj.built = true + return obj.sig(), nil } t1, exists := typ.Map[typ.Ord[1]] if !exists || t1 == nil { - return fmt.Errorf("second arg must be specified") + return nil, fmt.Errorf("second arg must be specified") } obj.Type = t1 // extracted vars type is now known! - return nil + obj.built = true + return obj.sig(), nil } // Validate makes sure we've built our struct properly. It is usually unused for // normal functions that users can use directly. func (obj *TemplateFunc) Validate() error { - if obj.Type == nil && !obj.NoVars { // build must be run first - return fmt.Errorf("type is still unspecified") + if !obj.built { + return fmt.Errorf("function wasn't built yet") } return nil } @@ -347,13 +349,8 @@ func (obj *TemplateFunc) Validate() error { // Info returns some static info about itself. func (obj *TemplateFunc) Info() *interfaces.Info { var sig *types.Type - if obj.NoVars { - str := fmt.Sprintf("func(%s str) str", templateArgNameTemplate) - sig = types.NewType(str) - - } else if obj.Type != nil { // don't panic if called speculatively - str := fmt.Sprintf("func(%s str, %s %s) str", templateArgNameTemplate, templateArgNameVars, obj.Type.String()) - sig = types.NewType(str) + if obj.built { + sig = obj.sig() // helper } return &interfaces.Info{ Pure: true, @@ -363,6 +360,17 @@ func (obj *TemplateFunc) Info() *interfaces.Info { } } +// helper +func (obj *TemplateFunc) sig() *types.Type { + if obj.Type != nil { // don't panic if called speculatively + str := fmt.Sprintf("func(%s str, %s %s) str", templateArgNameTemplate, templateArgNameVars, obj.Type.String()) + return types.NewType(str) + } + + str := fmt.Sprintf("func(%s str) str", templateArgNameTemplate) + return types.NewType(str) +} + // Init runs some startup code for this function. func (obj *TemplateFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/core/world/schedule_func.go b/lang/funcs/core/world/schedule_func.go index 2476a803b8..bb32d345d1 100644 --- a/lang/funcs/core/world/schedule_func.go +++ b/lang/funcs/core/world/schedule_func.go @@ -65,6 +65,8 @@ func init() { funcs.ModuleRegister(ModuleName, ScheduleFuncName, func() interfaces.Func { return &ScheduleFunc{} }) } +var _ interfaces.PolyFunc = &ScheduleFunc{} // ensure it meets this expectation + // ScheduleFunc is special function which determines where code should run in // the cluster. type ScheduleFunc struct { @@ -398,43 +400,43 @@ func (obj *ScheduleFunc) Polymorphisms(partialType *types.Type, partialValues [] // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *ScheduleFunc) Build(typ *types.Type) error { +func (obj *ScheduleFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 1 && len(typ.Ord) != 2 { - return fmt.Errorf("the schedule function needs either one or two args") + return nil, fmt.Errorf("the schedule function needs either one or two args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } if err := typ.Out.Cmp(types.NewType("[]str")); err != nil { - return errwrap.Wrapf(err, "return type must be a list of strings") + return nil, errwrap.Wrapf(err, "return type must be a list of strings") } tNamespace, exists := typ.Map[typ.Ord[0]] if !exists || tNamespace == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } if len(typ.Ord) == 1 { obj.Type = nil obj.built = true - return nil // done early, 2nd arg is absent! + return obj.sig(), nil // done early, 2nd arg is absent! } tOpts, exists := typ.Map[typ.Ord[1]] if !exists || tOpts == nil { - return fmt.Errorf("second argument was missing") + return nil, fmt.Errorf("second argument was missing") } if tOpts.Kind != types.KindStruct { - return fmt.Errorf("second argument must be of kind struct") + return nil, fmt.Errorf("second argument must be of kind struct") } validOpts := obj.validOpts() @@ -445,11 +447,11 @@ func (obj *ScheduleFunc) Build(typ *types.Type) error { t := tOpts.Map[name] value, exists := validOpts[name] if !exists { - return fmt.Errorf("unexpected opts field: `%s`", name) + return nil, fmt.Errorf("unexpected opts field: `%s`", name) } if err := t.Cmp(value); err != nil { - return errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) + return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) } } @@ -470,14 +472,14 @@ func (obj *ScheduleFunc) Build(typ *types.Type) error { // if it exists, check the type if err := t.Cmp(value); err != nil { - return errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) + return nil, errwrap.Wrapf(err, "expected different type for opts field: `%s`", name) } } } obj.Type = tOpts // type of opts struct, even an empty: `struct{}` obj.built = true - return nil + return obj.sig(), nil } // Validate tells us if the input struct takes a valid form. @@ -497,22 +499,28 @@ func (obj *ScheduleFunc) Validate() error { func (obj *ScheduleFunc) Info() *interfaces.Info { // It's important that you don't return a non-nil sig if this is called // before you're built. Type unification may call it opportunistically. - var typ *types.Type + var sig *types.Type if obj.built { - typ = types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace)) // simplest form - if obj.Type != nil { - typ = types.NewType(fmt.Sprintf("func(%s str, %s %s) []str", scheduleArgNameNamespace, scheduleArgNameOpts, obj.Type.String())) - } + sig = obj.sig() // helper } return &interfaces.Info{ Pure: false, // definitely false Memo: false, // output is list of hostnames chosen - Sig: typ, // func kind + Sig: sig, // func kind Err: obj.Validate(), } } +// helper +func (obj *ScheduleFunc) sig() *types.Type { + sig := types.NewType(fmt.Sprintf("func(%s str) []str", scheduleArgNameNamespace)) // simplest form + if obj.Type != nil { + sig = types.NewType(fmt.Sprintf("func(%s str, %s %s) []str", scheduleArgNameNamespace, scheduleArgNameOpts, obj.Type.String())) + } + return sig +} + // Init runs some startup code for this function. func (obj *ScheduleFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/dage/dage.go b/lang/funcs/dage/dage.go new file mode 100644 index 0000000000..e73d4adecf --- /dev/null +++ b/lang/funcs/dage/dage.go @@ -0,0 +1,1575 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Package dage implements a DAG function engine. +// TODO: can we rename this to something more interesting? +package dage + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "sync" + "time" + + "github.com/purpleidea/mgmt/engine" + "github.com/purpleidea/mgmt/lang/funcs/structs" + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/util/errwrap" +) + +// Engine implements a dag engine which lets us "run" a dag of functions, but +// also allows us to modify it while we are running. +type Engine struct { + // Name is the name used for the instance of the engine and in the graph + // that is held within it. + Name string + + Hostname string + World engine.World + + Debug bool + Logf func(format string, v ...interface{}) + + // Callback can be specified as an alternative to using the Stream + // method to get events. If the context on it is cancelled, then it must + // shutdown quickly, because this means we are closing and want to + // disconnect. Whether you want to respect that is up to you, but the + // engine will not be able to close until you do. If specified, and an + // error has occurred, it will set that error property. + Callback func(context.Context, error) + + graph *pgraph.Graph // guarded by graphMutex + table map[interfaces.Func]types.Value // guarded by tableMutex + state map[interfaces.Func]*state + + // graphMutex wraps access to the table map. + graphMutex *sync.Mutex // TODO: &sync.RWMutex{} ? + + // tableMutex wraps access to the table map. + tableMutex *sync.RWMutex + + // refCount keeps track of vertex and edge references across the entire + // graph. + refCount *RefCount + + // wgTxn blocks shutdown until the initial Txn has Reversed. + wgTxn *sync.WaitGroup + + // firstTxn checks to make sure wgTxn is only used for the first Txn. + firstTxn bool + + wg *sync.WaitGroup + + // pause/resume state machine signals + pauseChan chan struct{} + pausedChan chan struct{} + resumeChan chan struct{} + resumedChan chan struct{} + + // resend tracks which new nodes might need a new notification + resend map[interfaces.Func]struct{} + + // nodeWaitFns is a list of cleanup functions to run after we've begun + // resume, but before we've resumed completely. These are actions that + // we would like to do when paused from a deleteVertex operation, but + // that would deadlock if we did. + nodeWaitFns []func() + + // nodeWaitMutex wraps access to the nodeWaitFns list. + nodeWaitMutex *sync.Mutex + + // streamChan is used to send notifications to the outside world. + streamChan chan error + + loaded bool // are all of the funcs loaded? + loadedChan chan struct{} // funcs loaded signal + + startedChan chan struct{} // closes when Run() starts + + // wakeChan contains a message when someone has asked for us to wake up. + wakeChan chan struct{} + + // ag is the aggregation channel which cues up outgoing events. + ag chan error + + // leafSend specifies if we should do an ag send because we have + // activity at a leaf. + leafSend bool + + // isClosed tracks nodes that have closed. This list is purged as they + // are removed from the graph. + isClosed map[*state]struct{} + + // activity tracks nodes that are ready to send to ag. The main process + // loop decides if we have the correct set to do so. A corresponding + // value of true means we have regular activity, and a value of false + // means the node closed. + activity map[*state]struct{} + + // stateMutex wraps access to the isClosed and activity maps. + stateMutex *sync.Mutex + + // stats holds some statistics and other debugging information. + stats *stats // guarded by statsMutex + + // statsMutex wraps access to the stats data. + statsMutex *sync.RWMutex + + // graphvizMutex wraps access to the Graphviz method. + graphvizMutex *sync.Mutex + + // graphvizCount keeps a running tally of how many graphs we've + // generated. This is useful for displaying a sequence (timeline) of + // graphs in a linear order. + graphvizCount int64 + + // graphvizDirectory stores the generated path for outputting graphviz + // files if one is not specified at runtime. + graphvizDirectory string +} + +// Setup sets up the internal datastructures needed for this engine. +func (obj *Engine) Setup() error { + var err error + obj.graph, err = pgraph.NewGraph(obj.Name) + if err != nil { + return err + } + obj.table = make(map[interfaces.Func]types.Value) + obj.state = make(map[interfaces.Func]*state) + obj.graphMutex = &sync.Mutex{} // TODO: &sync.RWMutex{} ? + obj.tableMutex = &sync.RWMutex{} + + obj.refCount = (&RefCount{}).Init() + + obj.wgTxn = &sync.WaitGroup{} + + obj.wg = &sync.WaitGroup{} + + obj.pauseChan = make(chan struct{}) + obj.pausedChan = make(chan struct{}) + obj.resumeChan = make(chan struct{}) + obj.resumedChan = make(chan struct{}) + + obj.resend = make(map[interfaces.Func]struct{}) + + obj.nodeWaitFns = []func(){} + + obj.nodeWaitMutex = &sync.Mutex{} + + obj.streamChan = make(chan error) + obj.loadedChan = make(chan struct{}) + obj.startedChan = make(chan struct{}) + + obj.wakeChan = make(chan struct{}, 1) // hold up to one message + + obj.ag = make(chan error) + + obj.isClosed = make(map[*state]struct{}) + + obj.activity = make(map[*state]struct{}) + obj.stateMutex = &sync.Mutex{} + + obj.stats = &stats{ + runningList: make(map[*state]struct{}), + loadedList: make(map[*state]bool), + inputList: make(map[*state]int64), + } + obj.statsMutex = &sync.RWMutex{} + + obj.graphvizMutex = &sync.Mutex{} + return nil +} + +// Cleanup cleans up and frees memory and resources after everything is done. +func (obj *Engine) Cleanup() error { + obj.wg.Wait() // don't cleanup these before Run() finished + close(obj.pauseChan) // free + close(obj.pausedChan) + close(obj.resumeChan) + close(obj.resumedChan) + return nil +} + +// Txn returns a transaction that is suitable for adding and removing from the +// graph. You must run Setup before this method is called. +func (obj *Engine) Txn() interfaces.Txn { + if obj.refCount == nil { + panic("you must run setup before first use") + } + // The very first initial Txn must have a wait group to make sure if we + // shutdown (in error) that we can Reverse things before the Lock/Unlock + // loop shutsdown. + var free func() + if !obj.firstTxn { + obj.firstTxn = true + obj.wgTxn.Add(1) + free = func() { + obj.wgTxn.Done() + } + } + return (&graphTxn{ + Lock: obj.Lock, + Unlock: obj.Unlock, + GraphAPI: obj, + RefCount: obj.refCount, // reference counting + FreeFunc: free, + }).init() +} + +// addVertex is the lockless version of the AddVertex function. This is needed +// so that AddEdge can add two vertices within the same lock. +func (obj *Engine) addVertex(f interfaces.Func) error { + if _, exists := obj.state[f]; exists { + // don't err dupes, because it makes using the AddEdge API yucky + return nil + } + + // add some extra checks for easier debugging + if f == nil { + return fmt.Errorf("missing func") + } + if f.Info() == nil { + return fmt.Errorf("missing func info") + } + sig := f.Info().Sig + if sig == nil { + return fmt.Errorf("missing func sig") + } + if sig.Kind != types.KindFunc { + return fmt.Errorf("must be kind func") + } + if err := f.Validate(); err != nil { + return errwrap.Wrapf(err, "node did not Validate") + } + + input := make(chan types.Value) + output := make(chan types.Value) + txn := obj.Txn() + + // This is the one of two places where we modify this map. To avoid + // concurrent writes, we only do this when we're locked! Anywhere that + // can read where we are locked must have a mutex around it or do the + // lookup when we're in an unlocked state. + node := &state{ + Func: f, + name: f.String(), // cache a name to avoid locks + + input: input, + output: output, + txn: txn, + + running: false, + wg: &sync.WaitGroup{}, + + rwmutex: &sync.RWMutex{}, + } + + init := &interfaces.Init{ + Hostname: obj.Hostname, + Input: node.input, + Output: node.output, + Txn: node.txn, + World: obj.World, + Debug: obj.Debug, + Logf: func(format string, v ...interface{}) { + // safe Logf in case f.String contains %? chars... + s := f.String() + ": " + fmt.Sprintf(format, v...) + obj.Logf("%s", s) + }, + } + + if err := f.Init(init); err != nil { + return err + } + // only now, do we modify the graph + obj.state[f] = node + obj.graph.AddVertex(f) + return nil +} + +// AddVertex is the thread-safe way to add a vertex. You will need to call the +// engine Lock method before using this and the Unlock method afterwards. +func (obj *Engine) AddVertex(f interfaces.Func) error { + obj.graphMutex.Lock() + defer obj.graphMutex.Unlock() + if obj.Debug { + obj.Logf("Engine:AddVertex: %p %s", f, f) + } + + return obj.addVertex(f) // lockless version +} + +// AddEdge is the thread-safe way to add an edge. You will need to call the +// engine Lock method before using this and the Unlock method afterwards. This +// will automatically run AddVertex on both input vertices if they are not +// already part of the graph. You should only create DAG's as this function +// engine cannot handle cycles and this method will error if you cause a cycle. +func (obj *Engine) AddEdge(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) error { + obj.graphMutex.Lock() + defer obj.graphMutex.Unlock() + if obj.Debug { + obj.Logf("Engine:AddEdge %p %s: %p %s -> %p %s", fe, fe, f1, f1, f2, f2) + } + + // safety check to avoid cycles + g := obj.graph.Copy() + //g.AddVertex(f1) + //g.AddVertex(f2) + g.AddEdge(f1, f2, fe) + if _, err := g.TopologicalSort(); err != nil { + return err // not a dag + } + // if we didn't cycle, we can modify the real graph safely... + + // Does the graph already have these nodes in it? + hasf1 := obj.graph.HasVertex(f1) + //hasf2 := obj.graph.HasVertex(f2) + + if err := obj.addVertex(f1); err != nil { // lockless version + return err + } + if err := obj.addVertex(f2); err != nil { + // rollback f1 on error of f2 + obj.deleteVertex(f1) // ignore any error + return err + } + + // If f1 doesn't exist, let f1 (or it's incoming nodes) get the notify. + // If f2 is new, then it should get a new notification unless f1 is new. + // But there's no guarantee we didn't AddVertex(f2); AddEdge(f1, f2, e), + // so resend if f1 already exists. Otherwise it's not a new notification. + // previously: `if hasf1 && !hasf2` + if hasf1 { + obj.resend[f2] = struct{}{} // resend notification to me + } + + obj.graph.AddEdge(f1, f2, fe) // replaces any existing edge here + + // This shouldn't error, since the test graph didn't find a cycle. + if _, err := obj.graph.TopologicalSort(); err != nil { + // programming error + panic(err) // not a dag + } + + return nil +} + +// deleteVertex is the lockless version of the DeleteVertex function. This is +// needed so that AddEdge can add two vertices within the same lock. It needs +// deleteVertex so it can rollback the first one if the second addVertex fails. +func (obj *Engine) deleteVertex(f interfaces.Func) error { + node, exists := obj.state[f] + if !exists { + return fmt.Errorf("vertex %p %s doesn't exist", f, f) + } + + if node.running { + // cancel the running vertex + node.cancel() // cancel inner ctx + + // We store this work to be performed later on in the main loop + // because this Wait() might be blocked by a defer Commit, which + // is itself blocked because this deleteVertex operation is part + // of a Commit. + obj.nodeWaitMutex.Lock() + obj.nodeWaitFns = append(obj.nodeWaitFns, func() { + node.wg.Wait() // While waiting, the Stream might cause a new Reverse Commit + node.txn.Free() // Clean up when done! + obj.stateMutex.Lock() + delete(obj.isClosed, node) // avoid memory leak + obj.stateMutex.Unlock() + }) + obj.nodeWaitMutex.Unlock() + } + + // This is the one of two places where we modify this map. To avoid + // concurrent writes, we only do this when we're locked! Anywhere that + // can read where we are locked must have a mutex around it or do the + // lookup when we're in an unlocked state. + delete(obj.state, f) + obj.graph.DeleteVertex(f) + return nil +} + +// DeleteVertex is the thread-safe way to delete a vertex. You will need to call +// the engine Lock method before using this and the Unlock method afterwards. +func (obj *Engine) DeleteVertex(f interfaces.Func) error { + obj.graphMutex.Lock() + defer obj.graphMutex.Unlock() + if obj.Debug { + obj.Logf("Engine:DeleteVertex: %p %s", f, f) + } + + return obj.deleteVertex(f) // lockless version +} + +// DeleteEdge is the thread-safe way to delete an edge. You will need to call +// the engine Lock method before using this and the Unlock method afterwards. +func (obj *Engine) DeleteEdge(fe *interfaces.FuncEdge) error { + obj.graphMutex.Lock() + defer obj.graphMutex.Unlock() + if obj.Debug { + f1, f2, found := obj.graph.LookupEdge(fe) + if found { + obj.Logf("Engine:DeleteEdge: %p %s -> %p %s", f1, f1, f2, f2) + } else { + obj.Logf("Engine:DeleteEdge: not found %p %s", fe, fe) + } + } + + // Don't bother checking if edge exists first and don't error if it + // doesn't because it might have gotten deleted when a vertex did, and + // so there's no need to complain for nothing. + obj.graph.DeleteEdge(fe) + + return nil +} + +// HasVertex is the thread-safe way to check if a vertex exists in the graph. +// You will need to call the engine Lock method before using this and the Unlock +// method afterwards. +func (obj *Engine) HasVertex(f interfaces.Func) bool { + obj.graphMutex.Lock() // XXX: should this be a RLock? + defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + + return obj.graph.HasVertex(f) +} + +// LookupEdge is the thread-safe way to check which vertices (if any) exist +// between an edge in the graph. You will need to call the engine Lock method +// before using this and the Unlock method afterwards. +func (obj *Engine) LookupEdge(fe *interfaces.FuncEdge) (interfaces.Func, interfaces.Func, bool) { + obj.graphMutex.Lock() // XXX: should this be a RLock? + defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + + v1, v2, found := obj.graph.LookupEdge(fe) + if !found { + return nil, nil, found + } + f1, ok := v1.(interfaces.Func) + if !ok { + panic("not a Func") + } + f2, ok := v2.(interfaces.Func) + if !ok { + panic("not a Func") + } + return f1, f2, found +} + +// FindEdge is the thread-safe way to check which edge (if any) exists between +// two vertices in the graph. This is an important method in edge removal, +// because it's what you really need to know for DeleteEdge to work. Requesting +// a specific deletion isn't very sensical in this library when specified as the +// edge pointer, since we might replace it with a new edge that has new arg +// names. Instead, use this to look up what relationship you want, and then +// DeleteEdge to remove it. You will need to call the engine Lock method before +// using this and the Unlock method afterwards. +func (obj *Engine) FindEdge(f1, f2 interfaces.Func) *interfaces.FuncEdge { + obj.graphMutex.Lock() // XXX: should this be a RLock? + defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + + edge := obj.graph.FindEdge(f1, f2) + if edge == nil { + return nil + } + fe, ok := edge.(*interfaces.FuncEdge) + if !ok { + panic("edge is not a FuncEdge") + } + + return fe +} + +// Lock must be used before modifying the running graph. Make sure to Unlock +// when done. +// XXX: should Lock take a context if we want to bail mid-way? +// TODO: could we replace pauseChan with SubscribedSignal ? +func (obj *Engine) Lock() { // pause + select { + case obj.pauseChan <- struct{}{}: + } + //obj.rwmutex.Lock() // TODO: or should it go right before pauseChan? + + // waiting for the pause to move to paused... + select { + case <-obj.pausedChan: + } + // this mutex locks at start of Run() and unlocks at finish of Run() + obj.graphMutex.Unlock() // safe to make changes now +} + +// Unlock must be used after modifying the running graph. Make sure to Lock +// beforehand. +// XXX: should Unlock take a context if we want to bail mid-way? +func (obj *Engine) Unlock() { // resume + // this mutex locks at start of Run() and unlocks at finish of Run() + obj.graphMutex.Lock() // no more changes are allowed + select { + case obj.resumeChan <- struct{}{}: + } + //obj.rwmutex.Unlock() // TODO: or should it go right after resumedChan? + + // waiting for the resume to move to resumed... + select { + case <-obj.resumedChan: + } +} + +// wake sends a message to the wake queue to wake up the main process function +// which would otherwise spin unnecessarily. This can be called anytime, and +// doesn't hurt, it only wastes cpu if there's nothing to do. This does NOT ever +// block, and that's important so it can be called from anywhere. +func (obj *Engine) wake(name string) { + // The mutex guards the len check to avoid this function sending two + // messages down the channel, because the second would block if the + // consumer isn't fast enough. This mutex makes this method effectively + // asynchronous. + //obj.wakeMutex.Lock() + //defer obj.wakeMutex.Unlock() + //if len(obj.wakeChan) > 0 { // collapse duplicate, pending wake signals + // return + //} + select { + case obj.wakeChan <- struct{}{}: // send to chan of length 1 + if obj.Debug { + obj.Logf("wake sent from: %s", name) + } + default: // this is a cheap alternative to avoid the mutex altogether! + if obj.Debug { + obj.Logf("wake skip from: %s", name) + } + // skip sending, we already have a message pending! + } +} + +// runNodeWaitFns is a helper to run the cleanup nodeWaitFns list. It clears the +// list after it runs. +func (obj *Engine) runNodeWaitFns() { + // The lock is probably not needed here, but it won't hurt either. + obj.nodeWaitMutex.Lock() + defer obj.nodeWaitMutex.Unlock() + for _, fn := range obj.nodeWaitFns { + fn() + } + obj.nodeWaitFns = []func(){} // clear +} + +// process is the inner loop that runs through the entire graph. It can be +// called successively safely, as it is roughly idempotent, and is used to push +// values through the graph. If it is interrupted, it can pick up where it left +// off on the next run. This does however require it to re-check some things, +// but that is the price we pay for being always available to unblock. +// Importantly, re-running this resumes work in progress even if there was +// caching, and that if interrupted, it'll be queued again so as to not drop a +// wakeChan notification! We know we've read all the pending incoming values, +// because the Stream reader call wake(). +func (obj *Engine) process(ctx context.Context) (reterr error) { + defer func() { + // catch programming errors + if r := recover(); r != nil { + obj.Logf("Panic in process: %+v", r) + reterr = fmt.Errorf("panic in process: %+v", r) + } + }() + + // Toposort in dependency order. + topoSort, err := obj.graph.TopologicalSort() + if err != nil { + return err + } + + loaded := true // assume we emitted at least one value for now... + + outDegree := obj.graph.OutDegree() // map[Vertex]int + + for _, v := range topoSort { + f, ok := v.(interfaces.Func) + if !ok { + panic("not a Func") + } + node, exists := obj.state[f] + if !exists { + panic(fmt.Sprintf("missing node in iterate: %s", f)) + } + + out, exists := outDegree[f] + if !exists { + panic(fmt.Sprintf("missing out degree in iterate: %s", f)) + } + //outgoing := obj.graph.OutgoingGraphVertices(f) // []pgraph.Vertex + //node.isLeaf = len(outgoing) == 0 + node.isLeaf = out == 0 // store + + // TODO: the obj.loaded stuff isn't really consumed currently + node.rwmutex.RLock() + if !node.loaded { + loaded = false // we were wrong + } + node.rwmutex.RUnlock() + + // TODO: memoize since graph shape doesn't change in this loop! + incoming := obj.graph.IncomingGraphVertices(f) // []pgraph.Vertex + + // no incoming edges, so no incoming data + if len(incoming) == 0 || node.inputClosed { // we do this below + if !node.inputClosed { + node.inputClosed = true + close(node.input) + } + continue + } // else, process input data below... + + ready := true // assume all input values are ready for now... + inputClosed := true // assume all inputs have closed for now... + si := &types.Type{ + // input to functions are structs + Kind: types.KindStruct, + Map: node.Func.Info().Sig.Map, + Ord: node.Func.Info().Sig.Ord, + } + st := types.NewStruct(si) + // The above builds a struct with fields + // populated for each key (empty values) + // so we need to very carefully check if + // every field is received before we can + // safely send it downstream to an edge. + need := make(map[string]struct{}) // keys we need + for _, k := range node.Func.Info().Sig.Ord { + need[k] = struct{}{} + } + + for _, vv := range incoming { + ff, ok := vv.(interfaces.Func) + if !ok { + panic("not a Func") + } + obj.tableMutex.RLock() + value, exists := obj.table[ff] + obj.tableMutex.RUnlock() + if !exists { + ready = false // nope! + inputClosed = false // can't be, it's not even ready yet + break + } + // XXX: do we need a lock around reading obj.state? + fromNode, exists := obj.state[ff] + if !exists { + panic(fmt.Sprintf("missing node in notify: %s", ff)) + } + if !fromNode.outputClosed { + inputClosed = false // if any still open, then we are + } + + // set each arg, since one value + // could get used for multiple + // function inputs (shared edge) + args := obj.graph.Adjacency()[ff][f].(*interfaces.FuncEdge).Args + for _, arg := range args { + // populate struct + if err := st.Set(arg, value); err != nil { + //panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v", node, fromNode, err)) + keys := []string{} + for k := range st.Struct() { + keys = append(keys, k) + } + panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v, has: %v", node, fromNode, err, keys)) + } + if _, exists := need[arg]; !exists { + keys := []string{} + for k := range st.Struct() { + keys = append(keys, k) + } + // could be either a duplicate or an unwanted field (edge name) + panic(fmt.Sprintf("unexpected struct key on `%s` from `%s`: %v, has: %v", node, fromNode, err, keys)) + } + delete(need, arg) + } + } + + if !ready || len(need) != 0 { + continue // definitely continue, don't break here + } + + // previously it was closed, skip sending + if node.inputClosed { + continue + } + + // XXX: respect the info.Pure and info.Memo fields somewhere... + + // XXX: keep track of some state about who i sent to last before + // being interrupted so that I can avoid resending to some nodes + // if it's not necessary... + + // It's critical to avoid deadlock with this sending select that + // any events that could happen during this send can be + // preempted and that future executions of this function can be + // resumed. We must return with an error to let folks know that + // we were interrupted. + obj.Logf("send to func `%s`", node) + select { + case node.input <- st: // send to function + obj.statsMutex.Lock() + val, _ := obj.stats.inputList[node] // val is # or zero + obj.stats.inputList[node] = val + 1 // increment + obj.statsMutex.Unlock() + // pass + case <-node.ctx.Done(): // node died + obj.wake("node.ctx.Done()") // interrupted, so queue again + // XXX: can this happen now and should we continue or err? + return node.ctx.Err() + // continue // probably best to return and come finish later + case <-ctx.Done(): + obj.wake("node ctx.Done()") // interrupted, so queue again + return ctx.Err() + } + + // It's okay if this section gets preempted and we re-run this + // function. The worst that happens is we end up sending the + // same input data a second time. This means that we could in + // theory be causing unnecessary graph changes (and locks which + // cause preemption here) if nodes that cause locks aren't + // skipping duplicate/identical input values! + if inputClosed && !node.inputClosed { + node.inputClosed = true + close(node.input) + } + + // XXX: Do we need to somehow wait to make sure that node has + // the time to send at least one output? + // XXX: We could add a counter to each input that gets passed + // through the function... Eg: if we pass in 4, we should wait + // until a 4 comes out the output side. But we'd need to change + // the signature of func for this... + + } // end topoSort loop + + // It's okay if this section gets preempted and we re-run this bit here. + obj.loaded = loaded // this gets reset when graph adds new nodes + + if !loaded { + return nil + } + + // Check each leaf and make sure they're all ready to send, for us to + // send anything to ag channel. In addition, we need at least one send + // message from any of the valid isLeaf nodes. Since this only runs if + // everyone is loaded, we just need to check for activty leaf nodes. + obj.stateMutex.Lock() + for node := range obj.activity { + if obj.leafSend { + break // early + } + + // down here we need `true` activity! + if node.isLeaf { // calculated above in the previous loop + obj.leafSend = true + break + } + } + obj.activity = make(map[*state]struct{}) // clear + //clear(obj.activity) // new clear + + // This check happens here after the send loop to make sure one value + // got in and we didn't close it off too early. + for node := range obj.isClosed { // these are closed + node.outputClosed = true + } + obj.stateMutex.Unlock() + + if !obj.leafSend { + return nil + } + + select { + case obj.ag <- nil: // send to aggregate channel if we have events + obj.Logf("aggregated send") + obj.leafSend = false // reset + + case <-ctx.Done(): + obj.leafSend = true // since we skipped the ag send! + obj.wake("process ctx.Done()") // interrupted, so queue again + return ctx.Err() + + // XXX: should we even allow this default case? + //default: + // // exit if we're not ready to send to ag + // obj.leafSend = true // since we skipped the ag send! + // obj.wake("process default") // interrupted, so queue again + } + + return nil +} + +// Run kicks off the main engine. This takes a mutex. When we're "paused" the +// mutex is temporarily released until we "resume". Those operations transition +// with the engine Lock and Unlock methods. It is recommended to only add +// vertices to the engine after it's running. If you add them before Run, then +// Run will cause a Lock/Unlock to occur to cycle them in. Lock and Unlock race +// with the cancellation of this Run main loop. Make sure to only call one at a +// time. +func (obj *Engine) Run(ctx context.Context) (reterr error) { + obj.graphMutex.Lock() + defer obj.graphMutex.Unlock() + + // XXX: can the above defer get called while we are already unlocked? + // XXX: is it a possibility if we use <-Started() ? + + wg := &sync.WaitGroup{} + defer wg.Wait() + + defer func() { + // catch programming errors + if r := recover(); r != nil { + obj.Logf("Panic in Run: %+v", r) + reterr = fmt.Errorf("panic in Run: %+v", r) + } + }() + + ctx, cancel := context.WithCancel(ctx) // wrap parent + defer cancel() + + // Add a wait before the "started" signal runs so that Cleanup waits. + obj.wg.Add(1) + defer obj.wg.Done() + + // Send the start signal. + close(obj.startedChan) + + if n := obj.graph.NumVertices(); n > 0 { // hack to make the api easier + obj.Logf("graph contained %d vertices before Run", n) + wg.Add(1) + go func() { + defer wg.Done() + // kick the engine once to pull in any vertices from + // before we started running! + defer obj.Unlock() + obj.Lock() + }() + } + + once := &sync.Once{} + loadedSignal := func() { close(obj.loadedChan) } // only run once! + + // aggregate events channel + wg.Add(1) + go func() { + defer wg.Done() + defer close(obj.streamChan) + drain := false + for { + var err error + var ok bool + select { + case err, ok = <-obj.ag: // aggregated channel + if !ok { + return // channel shutdown + } + } + + if drain { + continue // no need to send more errors + } + + // TODO: check obj.loaded first? + once.Do(loadedSignal) + + // now send event... + if obj.Callback != nil { + // send stream signal (callback variant) + obj.Callback(ctx, err) + } else { + // send stream signal + select { + // send events or errors on streamChan + case obj.streamChan <- err: // send + case <-ctx.Done(): // when asked to exit + return + } + } + if err != nil { + cancel() // cancel the context! + //return // let the obj.ag channel drain + drain = true + } + } + }() + + // wgAg is a wait group that waits for all senders to the ag chan. + // Exceptionally, we don't close the ag channel until wgFor has also + // closed, because it can send to wg in process(). + wgAg := &sync.WaitGroup{} + wgFor := &sync.WaitGroup{} + + // We need to keep the main loop running until everyone else has shut + // down. When the top context closes, we wait for everyone to finish, + // and then we shut down this main context. + //mainCtx, mainCancel := context.WithCancel(ctx) // wrap parent + mainCtx, mainCancel := context.WithCancel(context.Background()) // DON'T wrap parent, close on your own terms + defer mainCancel() + + // close the aggregate channel when everyone is done with it... + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + } + + // don't wait and close ag before we're really done with Run() + wgAg.Wait() // wait for last ag user to close + obj.wgTxn.Wait() // wait for first txn as well + mainCancel() // only cancel after wgAg goroutines are done + wgFor.Wait() // wait for process loop to close before closing + close(obj.ag) // last one closes the ag channel + }() + + wgFn := &sync.WaitGroup{} // wg for process function runner + defer wgFn.Wait() // extra safety + + defer obj.runNodeWaitFns() // just in case + + wgFor.Add(1) // make sure we wait for the below process loop to exit... + defer wgFor.Done() + + // errProcess and processBreakFn are used to help exit following an err. + // This approach is needed because if we simply exited, we'd block the + // main loop below because various Stream functions are waiting on the + // Lock/Unlock cycle to be able to finish cleanly, shutdown, and unblock + // all the waitgroups so we can exit. + var errProcess error + var pausedProcess bool + processBreakFn := func(err error /*, paused bool*/) { + if err == nil { // a nil error won't cause ag to shutdown below + panic("expected error, not nil") + } + obj.Logf("process break") + select { + case obj.ag <- err: // send error to aggregate channel + case <-ctx.Done(): + } + cancel() // to unblock + //mainCancel() // NO! + errProcess = err // set above error + //pausedProcess = paused // set this inline directly + } + if obj.Debug { + defer obj.Logf("exited main loop") + } + // we start off "running", but we'll have an empty graph initially... + for { + + // After we've resumed, we can try to exit. (shortcut) + // NOTE: If someone calls Lock(), which would send to + // obj.pauseChan, it *won't* deadlock here because mainCtx is + // only closed when all the worker waitgroups close first! + select { + case <-mainCtx.Done(): // when asked to exit + return errProcess // we exit happily + default: + } + + // run through our graph, check for pause request occasionally + for { + pausedProcess = false // reset + // if we're in errProcess, we skip the process loop! + if errProcess != nil { + break // skip this process loop + } + + // Start the process run for this iteration of the loop. + ctxFn, cancelFn := context.WithCancel(context.Background()) + // we run cancelFn() below to cleanup! + var errFn error + chanFn := make(chan struct{}) // normal exit signal + wgFn.Add(1) + go func() { + defer wgFn.Done() + defer close(chanFn) // signal that I exited + for { + if obj.Debug { + obj.Logf("process...") + } + if errFn = obj.process(ctxFn); errFn != nil { // store + if errFn != context.Canceled { + obj.Logf("process end err: %+v...", errFn) + } + return + } + if obj.Debug { + obj.Logf("process end...") + } + // If process finishes without error, we + // should sit here and wait until we get + // run again from a wake-up, or we exit. + select { + case <-obj.wakeChan: // wait until something has actually woken up... + obj.Logf("process wakeup...") + // loop! + case <-ctxFn.Done(): + errFn = context.Canceled + return + } + } + }() + + chFn := false + chPause := false + ctxExit := false + select { + //case <-obj.wakeChan: + // this happens entirely in the process inner, inner loop now. + + case <-chanFn: // process exited on it's own in error! + chFn = true + + case <-obj.pauseChan: + obj.Logf("pausing...") + chPause = true + + case <-mainCtx.Done(): // when asked to exit + //return nil // we exit happily + ctxExit = true + } + + //fmt.Printf("chPause: %+v\n", chPause) // debug + //fmt.Printf("ctxExit: %+v\n", ctxExit) // debug + + cancelFn() // cancel the process function + wgFn.Wait() // wait for the process function to return + + pausedProcess = chPause // tell the below select + if errFn == nil { + // break on errors (needs to know if paused) + processBreakFn(fmt.Errorf("unexpected nil error in process")) + break + } + if errFn != nil && errFn != context.Canceled { + // break on errors (needs to know if paused) + processBreakFn(errwrap.Wrapf(errFn, "process error")) + break + } + //if errFn == context.Canceled { + // // ignore, we asked for it + //} + + if ctxExit { + return nil // we exit happily + } + if chPause { + break + } + if chFn && errFn == context.Canceled { // very rare case + return nil + } + + // programming error + //return fmt.Errorf("unhandled process state") + processBreakFn(fmt.Errorf("unhandled process state")) + break + } + // if we're in errProcess, we need to add back in the pauseChan! + if errProcess != nil && !pausedProcess { + select { + case <-obj.pauseChan: + obj.Logf("lower pausing...") + + // do we want this exit case? YES + case <-mainCtx.Done(): // when asked to exit + return errProcess + } + } + + // Toposort for paused workers. We run this before the actual + // pause completes, because the second we are paused, the graph + // could then immediately change. We don't need a lock in here + // because the mutex only unlocks when pause is complete below. + //topoSort1, err := obj.graph.TopologicalSort() + //if err != nil { + // return err + //} + //for _, v := range topoSort1 {} + + // pause is complete + // no exit case from here, must be fully running or paused... + select { + case obj.pausedChan <- struct{}{}: + obj.Logf("paused!") + } + + // + // the graph changes shape right here... we are locked right now + // + + // wait until resumed/unlocked + select { + case <-obj.resumeChan: + obj.Logf("resuming...") + } + + // Do any cleanup needed from delete vertex. Or do we? + // We've ascertained that while we want this stuff to shutdown, + // and while we also know that a Stream() function running is a + // part of what we're waiting for to exit, it doesn't matter + // that it exits now! This is actually causing a deadlock + // because the pending Stream exit, might be calling a new + // Reverse commit, which means we're deadlocked. It's safe for + // the Stream to keep running, all it might do is needlessly add + // a new value to obj.table which won't bother us since we won't + // even use it in process. We _do_ want to wait for all of these + // before the final exit, but we already have that in a defer. + //obj.runNodeWaitFns() + + // Toposort to run/resume workers. (Bottom of toposort first!) + topoSort2, err := obj.graph.TopologicalSort() + if err != nil { + return err + } + reversed := pgraph.Reverse(topoSort2) + for _, v := range reversed { + f, ok := v.(interfaces.Func) + if !ok { + panic("not a Func") + } + node, exists := obj.state[f] + if !exists { + panic(fmt.Sprintf("missing node in iterate: %s", f)) + } + + if node.running { // it's not a new vertex + continue + } + obj.loaded = false // reset this + node.running = true + + obj.statsMutex.Lock() + val, _ := obj.stats.inputList[node] // val is # or zero + obj.stats.inputList[node] = val // initialize to zero + obj.statsMutex.Unlock() + + innerCtx, innerCancel := context.WithCancel(ctx) // wrap parent (not mainCtx) + // we defer innerCancel() in the goroutine to cleanup! + node.ctx = innerCtx + node.cancel = innerCancel + + // run mainloop + wgAg.Add(1) + node.wg.Add(1) + go func(f interfaces.Func, node *state) { + defer node.wg.Done() + defer wgAg.Done() + defer node.cancel() // if we close, clean up and send the signal to anyone watching + if obj.Debug { + obj.Logf("Running func `%s`", node) + obj.statsMutex.Lock() + obj.stats.runningList[node] = struct{}{} + obj.stats.loadedList[node] = false + obj.statsMutex.Unlock() + } + + fn := func(nodeCtx context.Context) (reterr error) { + defer func() { + // catch programming errors + if r := recover(); r != nil { + obj.Logf("Panic in Stream of func `%s`: %+v", node, r) + reterr = fmt.Errorf("panic in Stream of func `%s`: %+v", node, r) + } + }() + return f.Stream(nodeCtx) + } + runErr := fn(node.ctx) // wrap with recover() + if obj.Debug { + obj.Logf("Exiting func `%s`", node) + obj.statsMutex.Lock() + delete(obj.stats.runningList, node) + obj.statsMutex.Unlock() + } + if runErr != nil { + obj.Logf("Erroring func `%s`: %+v", node, runErr) + // send to a aggregate channel + // the first to error will cause ag to + // shutdown, so make sure we can exit... + select { + case obj.ag <- runErr: // send to aggregate channel + case <-node.ctx.Done(): + } + } + // if node never loaded, then we error in the node.output loop! + }(f, node) + + // consume output + wgAg.Add(1) + node.wg.Add(1) + go func(f interfaces.Func, node *state) { + defer node.wg.Done() + defer wgAg.Done() + defer func() { + // We record the fact that output + // closed, so we can eventually close + // the downstream node's input. + obj.stateMutex.Lock() + obj.isClosed[node] = struct{}{} // closed! + obj.stateMutex.Unlock() + // TODO: is this wake necessary? + obj.wake("closed") // closed, so wake up + }() + + for value := range node.output { // read from channel + if value == nil { + // bug! + obj.Logf("func `%s` got nil value", node) + panic("got nil value") + } + + obj.tableMutex.RLock() + cached, exists := obj.table[f] + obj.tableMutex.RUnlock() + if !exists { // first value received + // RACE: do this AFTER value is present! + //node.loaded = true // not yet please + obj.Logf("func `%s` started", node) + } else if value.Cmp(cached) == nil { + // skip if new value is same as previous + // if this happens often, it *might* be + // a bug in the function implementation + // FIXME: do we need to disable engine + // caching when using hysteresis? + obj.Logf("func `%s` skipped", node) + continue + } + obj.tableMutex.Lock() + obj.table[f] = value // save the latest + obj.tableMutex.Unlock() + node.rwmutex.Lock() + node.loaded = true // set *after* value is in :) + //obj.Logf("func `%s` changed", node) + node.rwmutex.Unlock() + + obj.statsMutex.Lock() + obj.stats.loadedList[node] = true + obj.statsMutex.Unlock() + + // Send a message to tell our ag channel + // that we might have sent an aggregated + // message here. They should check if we + // are a leaf and if we glitch or not... + // Make sure we do this before the wake. + obj.stateMutex.Lock() + obj.activity[node] = struct{}{} // activity! + obj.stateMutex.Unlock() + + obj.wake("new value") // new value, so send wake up + + } // end for + + // no more output values are coming... + //obj.Logf("func `%s` stopped", node) + + // nodes that never loaded will cause the engine to hang + if !node.loaded { + select { + case obj.ag <- fmt.Errorf("func `%s` stopped before it was loaded", node): + case <-node.ctx.Done(): + return + } + } + + }(f, node) + + } // end for + + // Send new notifications in case any new edges are sending away + // to these... They might have already missed the notifications! + for k := range obj.resend { // resend TO these! + node, exists := obj.state[k] + if !exists { + continue + } + // Run as a goroutine to avoid erroring in parent thread. + wg.Add(1) + go func(node *state) { + defer wg.Done() + obj.Logf("resend to func `%s`", node) + obj.wake("resend") // new value, so send wake up + }(node) + } + obj.resend = make(map[interfaces.Func]struct{}) // reset + + // now check their states... + //for _, v := range reversed { + // v, ok := v.(interfaces.Func) + // if !ok { + // panic("not a Func") + // } + // // wait for startup? + // close(obj.state[v].startup) XXX: once? + //} + + // resume is complete + // no exit case from here, must be fully running or paused... + select { + case obj.resumedChan <- struct{}{}: + obj.Logf("resumed!") + } + + } // end for +} + +// Stream returns a channel that you can follow to get aggregated graph events. +// Do not block reading from this channel as you can hold up the entire engine. +func (obj *Engine) Stream() <-chan error { + return obj.streamChan +} + +// Loaded returns a channel that closes when the function engine loads. +func (obj *Engine) Loaded() <-chan struct{} { + return obj.loadedChan +} + +// Table returns a copy of the populated data table of values. We return a copy +// because since these values are constantly changing, we need an atomic +// snapshot to present to the consumer of this API. +// TODO: is this globally glitch consistent? +// TODO: do we need an API to return a single value? (wrapped in read locks) +func (obj *Engine) Table() map[interfaces.Func]types.Value { + obj.tableMutex.RLock() + defer obj.tableMutex.RUnlock() + table := make(map[interfaces.Func]types.Value) + for k, v := range obj.table { + //table[k] = v.Copy() // TODO: do we need to copy these values? + table[k] = v + } + return table +} + +// Apply is similar to Table in that it gives you access to the internal output +// table of data, the difference being that it instead passes this information +// to a function of your choosing and holds a read/write lock during the entire +// time that your function is synchronously executing. If you use this function +// to spawn any goroutines that read or write data, then you're asking for a +// panic. +// XXX: does this need to be a Lock? Can it be an RLock? Check callers! +func (obj *Engine) Apply(fn func(map[interfaces.Func]types.Value) error) error { + // XXX: does this need to be a Lock? Can it be an RLock? Check callers! + obj.tableMutex.Lock() // differs from above RLock around obj.table + defer obj.tableMutex.Unlock() + table := make(map[interfaces.Func]types.Value) + for k, v := range obj.table { + //table[k] = v.Copy() // TODO: do we need to copy these values? + table[k] = v + } + + return fn(table) +} + +// Started returns a channel that closes when the Run function finishes starting +// up. This is useful so that we can wait before calling any of the mutex things +// that would normally panic if Run wasn't started up first. +func (obj *Engine) Started() <-chan struct{} { + return obj.startedChan +} + +// NumVertices returns the number of vertices in the current graph. +func (obj *Engine) NumVertices() int { + // XXX: would this deadlock if we added this? + //obj.graphMutex.Lock() // XXX: should this be a RLock? + //defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + return obj.graph.NumVertices() +} + +// Stats returns some statistics in a human-readable form. +func (obj *Engine) Stats() string { + defer obj.statsMutex.RUnlock() + obj.statsMutex.RLock() + + return obj.stats.String() +} + +// Graphviz writes out the diagram of a graph to be used for visualization and +// debugging. You must not modify the graph (eg: during Lock) when calling this +// method. +func (obj *Engine) Graphviz(dir string) error { + // XXX: would this deadlock if we added this? + //obj.graphMutex.Lock() // XXX: should this be a RLock? + //defer obj.graphMutex.Unlock() // XXX: should this be an RUnlock? + + obj.graphvizMutex.Lock() + defer obj.graphvizMutex.Unlock() + + obj.graphvizCount++ // increment + + if dir == "" { + dir = obj.graphvizDirectory + } + if dir == "" { // XXX: hack for ergonomics + d := time.Now().UnixMilli() + dir = fmt.Sprintf("/tmp/dage-graphviz-%s-%d/", obj.Name, d) + obj.graphvizDirectory = dir + } + if !strings.HasSuffix(dir, "/") { + return fmt.Errorf("dir must end with a slash") + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + dashedEdges, err := pgraph.NewGraph("dashedEdges") + if err != nil { + return err + } + for _, v1 := range obj.graph.Vertices() { + // if it's a ChannelBasedSinkFunc... + if cb, ok := v1.(*structs.ChannelBasedSinkFunc); ok { + // ...then add a dashed edge to its output + dashedEdges.AddEdge(v1, cb.Target, &pgraph.SimpleEdge{ + Name: "channel", // secret channel + }) + } + // if it's a ChannelBasedSourceFunc... + if cb, ok := v1.(*structs.ChannelBasedSourceFunc); ok { + // ...then add a dashed edge from its input + dashedEdges.AddEdge(cb.Source, v1, &pgraph.SimpleEdge{ + Name: "channel", // secret channel + }) + } + } + + gv := &pgraph.Graphviz{ + Name: obj.graph.GetName(), + Filename: fmt.Sprintf("%s/%d.dot", dir, obj.graphvizCount), + Graphs: map[*pgraph.Graph]*pgraph.GraphvizOpts{ + obj.graph: nil, + dashedEdges: { + Style: "dashed", + }, + }, + } + + if err := gv.Exec(); err != nil { + return err + } + return nil +} + +// state tracks some internal vertex-specific state information. +type state struct { + Func interfaces.Func + name string // cache a name here for safer concurrency + + input chan types.Value // the top level type must be a struct + output chan types.Value + txn interfaces.Txn // API of graphTxn struct to pass to each function + + //init bool // have we run Init on our func? + //ready bool // has it received all the args it needs at least once? + loaded bool // has the func run at least once ? + inputClosed bool // is our input closed? + outputClosed bool // is our output closed? + + isLeaf bool // is my out degree zero? + + running bool + wg *sync.WaitGroup + ctx context.Context // per state ctx (inner ctx) + cancel func() // cancel above inner ctx + + rwmutex *sync.RWMutex // concurrency guard for reading/modifying this state +} + +// String implements the fmt.Stringer interface for pretty printing! +func (obj *state) String() string { + if obj.name != "" { + return obj.name + } + + return obj.Func.String() +} + +// stats holds some statistics and other debugging information. +type stats struct { + + // runningList keeps track of which nodes are still running. + runningList map[*state]struct{} + + // loadedList keeps track of which nodes have loaded. + loadedList map[*state]bool + + // inputList keeps track of the number of inputs each node received. + inputList map[*state]int64 +} + +// String implements the fmt.Stringer interface for printing out our collected +// statistics! +func (obj *stats) String() string { + // XXX: just build the lock into *stats instead of into our dage obj + s := "stats:\n" + { + s += "\trunning:\n" + names := []string{} + for k := range obj.runningList { + names = append(names, k.String()) + } + sort.Strings(names) + for _, name := range names { + s += fmt.Sprintf("\t * %s\n", name) + } + } + { + nodes := []*state{} + for k := range obj.loadedList { + nodes = append(nodes, k) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].String() < nodes[j].String() }) + + s += "\tloaded:\n" + for _, node := range nodes { + if !obj.loadedList[node] { + continue + } + s += fmt.Sprintf("\t * %s\n", node) + } + + s += "\tnot loaded:\n" + for _, node := range nodes { + if obj.loadedList[node] { + continue + } + s += fmt.Sprintf("\t * %s\n", node) + } + } + { + s += "\tinput count:\n" + nodes := []*state{} + for k := range obj.inputList { + nodes = append(nodes, k) + } + //sort.Slice(nodes, func(i, j int) bool { return nodes[i].String() < nodes[j].String() }) + sort.Slice(nodes, func(i, j int) bool { return obj.inputList[nodes[i]] < obj.inputList[nodes[j]] }) + for _, node := range nodes { + s += fmt.Sprintf("\t * (%d) %s\n", obj.inputList[node], node) + } + } + return s +} diff --git a/lang/funcs/dage/dage_test.go b/lang/funcs/dage/dage_test.go new file mode 100644 index 0000000000..9fcf44dc21 --- /dev/null +++ b/lang/funcs/dage/dage_test.go @@ -0,0 +1,793 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package dage + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util" +) + +type testFunc struct { + Name string + Type *types.Type + Func func(types.Value) (types.Value, error) + Meta *meta + + value types.Value + init *interfaces.Init +} + +func (obj *testFunc) String() string { return obj.Name } + +func (obj *testFunc) Info() *interfaces.Info { + return &interfaces.Info{ + Pure: true, + Memo: false, // TODO: should this be something we specify here? + Sig: obj.Type, + Err: obj.Validate(), + } +} + +func (obj *testFunc) Validate() error { + if obj.Meta == nil { + return fmt.Errorf("test case error: did you add the vertex to the vertices list?") + } + return nil +} + +func (obj *testFunc) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +func (obj *testFunc) Stream(ctx context.Context) error { + defer close(obj.init.Output) // the sender closes + defer obj.init.Logf("stream closed") + obj.init.Logf("stream startup") + + // make some placeholder value because obj.value is nil + constValue, err := types.ValueOfGolang("hello") + if err != nil { + return err // unlikely + } + + for { + select { + case input, ok := <-obj.init.Input: + if !ok { + obj.init.Logf("stream input closed") + obj.init.Input = nil // don't get two closes + // already sent one value, so we can shutdown + if obj.value != nil { + return nil // can't output any more + } + + obj.value = constValue + } else { + obj.init.Logf("stream got input type(%T) value: (%+v)", input, input) + if obj.Func == nil { + obj.value = constValue + } + + if obj.Func != nil { + //obj.init.Logf("running internal function...") + v, err := obj.Func(input) // run me! + if err != nil { + return err + } + obj.value = v + } + } + + case <-ctx.Done(): + return nil + } + + select { + case obj.init.Output <- obj.value: // send anything + // add some monitoring... + obj.Meta.wg.Add(1) + go func() { + // no need for lock here + defer obj.Meta.wg.Done() + if obj.Meta.debug { + obj.Meta.logf("sending an internal event!") + } + + select { + case obj.Meta.Events[obj.Name] <- struct{}{}: + case <-obj.Meta.ctx.Done(): + } + }() + + case <-ctx.Done(): + return nil + } + } +} + +type meta struct { + EventCount int + Event chan struct{} + Events map[string]chan struct{} + + ctx context.Context + wg *sync.WaitGroup + mutex *sync.Mutex + + debug bool + logf func(format string, v ...interface{}) +} + +func (obj *meta) Lock() { obj.mutex.Lock() } +func (obj *meta) Unlock() { obj.mutex.Unlock() } + +type dageTestOp func(*Engine, interfaces.Txn, *meta) error + +func TestDageTable(t *testing.T) { + + type test struct { // an individual test + name string + vertices []interfaces.Func + actions []dageTestOp + } + testCases := []test{} + { + testCases = append(testCases, test{ + name: "empty graph", + vertices: []interfaces.Func{}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + time.Sleep(1 * time.Second) // XXX: unfortunate + defer engine.Unlock() + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + // We don't expect an empty graph to send events. + if meta.EventCount != 0 { + return fmt.Errorf("got too many stream events") + } + return nil + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + + testCases = append(testCases, test{ + name: "simple add vertex", + vertices: []interfaces.Func{f1}, // so the test engine can pass in debug/observability handles + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.AddVertex(f1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get any stream events") + } + return nil + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + // e1 arg name must match incoming edge to it + f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "simple add edge", + vertices: []interfaces.Func{f1, f2}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.AddVertex(f1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + engine.Lock() + defer engine.Unlock() + // This newly added node should get a notification after it starts. + return engine.AddEdge(f1, f2, e1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 2 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + }, + }) + } + { + // diamond + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + f3 := &testFunc{Name: "f3", Type: types.NewType("func(e2 str) str")} + f4 := &testFunc{Name: "f4", Type: types.NewType("func(e3 str, e4 str) str")} + e1 := testEdge("e1") + e2 := testEdge("e2") + e3 := testEdge("e3") + e4 := testEdge("e4") + + testCases = append(testCases, test{ + name: "simple add multiple edges", + vertices: []interfaces.Func{f1, f2, f3, f4}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.AddVertex(f1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + if err := engine.AddEdge(f1, f2, e1); err != nil { + return err + } + if err := engine.AddEdge(f1, f3, e2); err != nil { + return err + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + if err := engine.AddEdge(f2, f4, e3); err != nil { + return err + } + if err := engine.AddEdge(f3, f4, e4); err != nil { + return err + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + //meta.Lock() + //defer meta.Unlock() + num := 1 + for { + if num == 0 { + break + } + select { + case _, ok := <-meta.Event: + if !ok { + return fmt.Errorf("unexpectedly channel close") + } + num-- + if meta.debug { + meta.logf("got an event!") + } + case <-meta.ctx.Done(): + return meta.ctx.Err() + } + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + //meta.Lock() + //defer meta.Unlock() + num := 1 + for { + if num == 0 { + break + } + bt := util.BlockedTimer{Seconds: 2} + defer bt.Cancel() + bt.Printf("waiting for f4...\n") + select { + case _, ok := <-meta.Events["f4"]: + bt.Cancel() + if !ok { + return fmt.Errorf("unexpectedly channel close") + } + num-- + if meta.debug { + meta.logf("got an event from f4!") + } + case <-meta.ctx.Done(): + return meta.ctx.Err() + } + } + return nil + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + + testCases = append(testCases, test{ + name: "simple add/delete vertex", + vertices: []interfaces.Func{f1}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.AddVertex(f1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + + //meta.Lock() + //defer meta.Unlock() + if meta.debug { + meta.logf("about to delete vertex f1!") + defer meta.logf("done deleting vertex f1!") + } + + return engine.DeleteVertex(f1) + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + // e1 arg name must match incoming edge to it + f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "simple add/delete edge", + vertices: []interfaces.Func{f1, f2}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.AddVertex(f1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + engine.Lock() + defer engine.Unlock() + // This newly added node should get a notification after it starts. + return engine.AddEdge(f1, f2, e1) + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 2 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + engine.Lock() + defer engine.Unlock() + return engine.DeleteEdge(e1) + }, + }, + }) + } + + // the following tests use the txn instead of direct locks + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + + testCases = append(testCases, test{ + name: "txn simple add vertex", + vertices: []interfaces.Func{f1}, // so the test engine can pass in debug/observability handles + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddVertex(f1).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get any stream events") + } + return nil + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + // e1 arg name must match incoming edge to it + f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "txn simple add edge", + vertices: []interfaces.Func{f1, f2}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddVertex(f1).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + // This newly added node should get a notification after it starts. + return txn.AddEdge(f1, f2, e1).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 2 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + }, + }) + } + { + // diamond + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + f3 := &testFunc{Name: "f3", Type: types.NewType("func(e2 str) str")} + f4 := &testFunc{Name: "f4", Type: types.NewType("func(e3 str, e4 str) str")} + e1 := testEdge("e1") + e2 := testEdge("e2") + e3 := testEdge("e3") + e4 := testEdge("e4") + + testCases = append(testCases, test{ + name: "txn simple add multiple edges", + vertices: []interfaces.Func{f1, f2, f3, f4}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddVertex(f1).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddEdge(f1, f2, e1).AddEdge(f1, f3, e2).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddEdge(f2, f4, e3).AddEdge(f3, f4, e4).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + //meta.Lock() + //defer meta.Unlock() + num := 1 + for { + if num == 0 { + break + } + select { + case _, ok := <-meta.Event: + if !ok { + return fmt.Errorf("unexpectedly channel close") + } + num-- + if meta.debug { + meta.logf("got an event!") + } + case <-meta.ctx.Done(): + return meta.ctx.Err() + } + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + //meta.Lock() + //defer meta.Unlock() + num := 1 + for { + if num == 0 { + break + } + bt := util.BlockedTimer{Seconds: 2} + defer bt.Cancel() + bt.Printf("waiting for f4...\n") + select { + case _, ok := <-meta.Events["f4"]: + bt.Cancel() + if !ok { + return fmt.Errorf("unexpectedly channel close") + } + num-- + if meta.debug { + meta.logf("got an event from f4!") + } + case <-meta.ctx.Done(): + return meta.ctx.Err() + } + } + return nil + }, + }, + }) + } + { + f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + + testCases = append(testCases, test{ + name: "txn simple add/delete vertex", + vertices: []interfaces.Func{f1}, + actions: []dageTestOp{ + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + return txn.AddVertex(f1).Commit() + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + time.Sleep(1 * time.Second) // XXX: unfortunate + meta.Lock() + defer meta.Unlock() + if meta.EventCount < 1 { + return fmt.Errorf("didn't get enough stream events") + } + return nil + }, + func(engine *Engine, txn interfaces.Txn, meta *meta) error { + //meta.Lock() + //defer meta.Unlock() + if meta.debug { + meta.logf("about to delete vertex f1!") + defer meta.logf("done deleting vertex f1!") + } + + return txn.DeleteVertex(f1).Commit() + }, + }, + }) + } + //{ + // f1 := &testFunc{Name: "f1", Type: types.NewType("func() str")} + // // e1 arg name must match incoming edge to it + // f2 := &testFunc{Name: "f2", Type: types.NewType("func(e1 str) str")} + // e1 := testEdge("e1") + // + // testCases = append(testCases, test{ + // name: "txn simple add/delete edge", + // vertices: []interfaces.Func{f1, f2}, + // actions: []dageTestOp{ + // func(engine *Engine, txn interfaces.Txn, meta *meta) error { + // return txn.AddVertex(f1).Commit() + // }, + // func(engine *Engine, txn interfaces.Txn, meta *meta) error { + // time.Sleep(1 * time.Second) // XXX: unfortunate + // // This newly added node should get a notification after it starts. + // return txn.AddEdge(f1, f2, e1).Commit() + // }, + // func(engine *Engine, txn interfaces.Txn, meta *meta) error { + // time.Sleep(1 * time.Second) // XXX: unfortunate + // meta.Lock() + // defer meta.Unlock() + // if meta.EventCount < 2 { + // return fmt.Errorf("didn't get enough stream events") + // } + // return nil + // }, + // func(engine *Engine, txn interfaces.Txn, meta *meta) error { + // return txn.DeleteEdge(e1).Commit() // XXX: not implemented + // }, + // }, + // }) + //} + + if testing.Short() { + t.Logf("available tests:") + } + names := []string{} + for index, tc := range testCases { // run all the tests + if tc.name == "" { + t.Errorf("test #%d: not named", index) + continue + } + if util.StrInList(tc.name, names) { + t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) + continue + } + names = append(names, tc.name) + + //if index != 3 { // hack to run a subset (useful for debugging) + //if tc.name != "simple txn" { + // continue + //} + + testName := fmt.Sprintf("test #%d (%s)", index, tc.name) + if testing.Short() { // make listing tests easier + t.Logf("%s", testName) + continue + } + t.Run(testName, func(t *testing.T) { + name, vertices, actions := tc.name, tc.vertices, tc.actions + + t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) + + //logf := func(format string, v ...interface{}) { + // t.Logf(fmt.Sprintf("test #%d", index)+": "+format, v...) + //} + + //now := time.Now() + + wg := &sync.WaitGroup{} + defer wg.Wait() // defer is correct b/c we're in a func! + + min := 5 * time.Second // approx min time needed for the test + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if deadline, ok := t.Deadline(); ok { + d := deadline.Add(-min) + //t.Logf(" now: %+v", now) + //t.Logf(" d: %+v", d) + newCtx, cancel := context.WithDeadline(ctx, d) + ctx = newCtx + defer cancel() + } + + debug := testing.Verbose() // set via the -test.v flag to `go test` + + meta := &meta{ + Event: make(chan struct{}), + Events: make(map[string]chan struct{}), + + ctx: ctx, + wg: &sync.WaitGroup{}, + mutex: &sync.Mutex{}, + + debug: debug, + logf: func(format string, v ...interface{}) { + // safe Logf in case f.String contains %? chars... + s := fmt.Sprintf(format, v...) + t.Logf("%s", s) + }, + } + defer meta.wg.Wait() + + for _, f := range vertices { + testFunc, ok := f.(*testFunc) + if !ok { + t.Errorf("bad test function: %+v", f) + return + } + meta.Events[testFunc.Name] = make(chan struct{}) + testFunc.Meta = meta // add the handle + } + + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + t.Logf("cancelling test...") + } + }() + + engine := &Engine{ + Name: "dage", + + Debug: debug, + Logf: t.Logf, + } + + if err := engine.Setup(); err != nil { + t.Errorf("could not setup engine: %+v", err) + return + } + defer engine.Cleanup() + + wg.Add(1) + go func() { + defer wg.Done() + if err := engine.Run(ctx); err != nil { + t.Errorf("error while running engine: %+v", err) + return + } + t.Logf("engine shutdown cleanly...") + }() + + <-engine.Started() // wait for startup (will not block forever) + + txn := engine.Txn() + defer txn.Free() // remember to call Free() + + wg.Add(1) + go func() { + defer wg.Done() + ch := engine.Stream() + for { + select { + case err, ok := <-ch: // channel must close to shutdown + if !ok { + return + } + meta.Lock() + meta.EventCount++ + meta.Unlock() + meta.wg.Add(1) + go func() { + // no need for lock here + defer meta.wg.Done() + if meta.debug { + meta.logf("sending an event!") + } + select { + case meta.Event <- struct{}{}: + case <-meta.ctx.Done(): + } + }() + if err != nil { + t.Errorf("graph error event: %v", err) + continue + } + t.Logf("graph stream event!") + } + } + }() + + // Run a list of actions. Any error kills it all. + t.Logf("starting actions...") + for i, action := range actions { + t.Logf("running action %d...", i+1) + if err := action(engine, txn, meta); err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: action #%d failed with: %+v", index, i, err) + break // so that cancel runs + } + } + + t.Logf("test done...") + cancel() + }) + } + + if testing.Short() { + t.Skip("skipping all tests...") + } +} diff --git a/lang/funcs/dage/ref.go b/lang/funcs/dage/ref.go new file mode 100644 index 0000000000..74cd80d2c4 --- /dev/null +++ b/lang/funcs/dage/ref.go @@ -0,0 +1,282 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Package dage implements a DAG function engine. +// TODO: can we rename this to something more interesting? +package dage + +import ( + "fmt" + "sync" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/util/errwrap" +) + +// RefCount keeps track of vertex and edge references across the entire graph. +// Make sure to lock access somehow, ideally with the provided Locker interface. +type RefCount struct { + // mutex locks this database for read or write. + mutex *sync.Mutex + + // vertices is a reference count of the number of vertices used. + vertices map[interfaces.Func]int64 + + // edges is a reference count of the number of edges used. + edges map[*RefCountEdge]int64 // TODO: hash *RefCountEdge as a key instead +} + +// RefCountEdge is a virtual "hash" entry for the RefCount edges map key. +type RefCountEdge struct { + f1 interfaces.Func + f2 interfaces.Func + arg string +} + +// String prints a representation of the references held. +func (obj *RefCount) String() string { + s := "" + s += fmt.Sprintf("vertices (%d):\n", len(obj.vertices)) + for vertex, count := range obj.vertices { + s += fmt.Sprintf("\tvertex (%d): %p %s\n", count, vertex, vertex) + } + s += fmt.Sprintf("edges (%d):\n", len(obj.edges)) + for edge, count := range obj.edges { + s += fmt.Sprintf("\tedge (%d): %p %s -> %p %s # %s\n", count, edge.f1, edge.f1, edge.f2, edge.f2, edge.arg) + } + return s +} + +// Init must be called to initialized the struct before first use. +func (obj *RefCount) Init() *RefCount { + obj.mutex = &sync.Mutex{} + obj.vertices = make(map[interfaces.Func]int64) + obj.edges = make(map[*RefCountEdge]int64) + return obj // return self so it can be called in a chain +} + +// Lock the mutex that should be used when reading or writing from this. +func (obj *RefCount) Lock() { obj.mutex.Lock() } + +// Unlock the mutex that should be used when reading or writing from this. +func (obj *RefCount) Unlock() { obj.mutex.Unlock() } + +// VertexInc increments the reference count for the input vertex. It returns +// true if the reference count for this vertex was previously undefined or zero. +// True usually means we'd want to actually add this vertex now. If you attempt +// to increment a vertex which already has a less than zero count, then this +// will panic. This situation is likely impossible unless someone modified the +// reference counting struct directly. +func (obj *RefCount) VertexInc(f interfaces.Func) bool { + count, _ := obj.vertices[f] + obj.vertices[f] = count + 1 + if count == -1 { // unlikely, but catch any bugs + panic("negative reference count") + } + return count == 0 +} + +// VertexDec decrements the reference count for the input vertex. It returns +// true if the reference count for this vertex is now zero. True usually means +// we'd want to actually remove this vertex now. If you attempt to decrement a +// vertex which already has a zero count, then this will panic. +func (obj *RefCount) VertexDec(f interfaces.Func) bool { + count, _ := obj.vertices[f] + obj.vertices[f] = count - 1 + if count == 0 { + panic("negative reference count") + } + return count == 1 // now it's zero +} + +// EdgeInc increments the reference count for the input edge. It adds a +// reference for each arg name in the edge. Since this also increments the +// references for the two input vertices, it returns the corresponding two +// boolean values for these calls. (This function makes two calls to VertexInc.) +func (obj *RefCount) EdgeInc(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) (bool, bool) { + for _, arg := range fe.Args { // ref count each arg + r := obj.makeEdge(f1, f2, arg) + count := obj.edges[r] + obj.edges[r] = count + 1 + if count == -1 { // unlikely, but catch any bugs + panic("negative reference count") + } + } + + return obj.VertexInc(f1), obj.VertexInc(f2) +} + +// EdgeDec decrements the reference count for the input edge. It removes a +// reference for each arg name in the edge. Since this also decrements the +// references for the two input vertices, it returns the corresponding two +// boolean values for these calls. (This function makes two calls to VertexDec.) +func (obj *RefCount) EdgeDec(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) (bool, bool) { + for _, arg := range fe.Args { // ref count each arg + r := obj.makeEdge(f1, f2, arg) + count := obj.edges[r] + obj.edges[r] = count - 1 + if count == 0 { + panic("negative reference count") + } + } + + return obj.VertexDec(f1), obj.VertexDec(f2) +} + +// FreeVertex removes exactly one entry from the Vertices list or it errors. +func (obj *RefCount) FreeVertex(f interfaces.Func) error { + if count, exists := obj.vertices[f]; !exists || count != 0 { + return fmt.Errorf("no vertex of count zero found") + } + delete(obj.vertices, f) + return nil +} + +// FreeEdge removes exactly one entry from the Edges list or it errors. +func (obj *RefCount) FreeEdge(f1, f2 interfaces.Func, arg string) error { + found := []*RefCountEdge{} + for k, count := range obj.edges { + //if k == nil { // programming error + // continue + //} + if k.f1 == f1 && k.f2 == f2 && k.arg == arg && count == 0 { + found = append(found, k) + } + } + if len(found) > 1 { + return fmt.Errorf("inconsistent ref count for edge") + } + if len(found) == 0 { + return fmt.Errorf("no edge of count zero found") + } + delete(obj.edges, found[0]) // delete from map + return nil +} + +// GC runs the garbage collector on any zeroed references. Note the distinction +// between count == 0 (please delete now) and absent from the map. +func (obj *RefCount) GC(graphAPI interfaces.GraphAPI) error { + // debug + //fmt.Printf("start refs\n%s", obj.String()) + //defer func() { fmt.Printf("end refs\n%s", obj.String()) }() + free := make(map[interfaces.Func]map[interfaces.Func][]string) // f1 -> f2 + for x, count := range obj.edges { + if count != 0 { // we only care about freed things + continue + } + if _, exists := free[x.f1]; !exists { + free[x.f1] = make(map[interfaces.Func][]string) + } + if _, exists := free[x.f1][x.f2]; !exists { + free[x.f1][x.f2] = []string{} + } + free[x.f1][x.f2] = append(free[x.f1][x.f2], x.arg) // exists as refcount zero + } + + // These edges have a refcount of zero. + for f1, x := range free { + for f2, args := range x { + for _, arg := range args { + edge := graphAPI.FindEdge(f1, f2) + // any errors here are programming errors + if edge == nil { + return fmt.Errorf("missing edge from %p %s -> %p %s", f1, f1, f2, f2) + } + + once := false // sanity check + newArgs := []string{} + for _, a := range edge.Args { + if arg == a { + if once { + // programming error, duplicate arg + return fmt.Errorf("duplicate arg (%s) in edge", arg) + } + once = true + continue + } + newArgs = append(newArgs, a) + } + + if len(edge.Args) == 1 { // edge gets deleted + if a := edge.Args[0]; a != arg { // one arg + return fmt.Errorf("inconsistent arg: %s != %s", a, arg) + } + + if err := graphAPI.DeleteEdge(edge); err != nil { + return errwrap.Wrapf(err, "edge deletion error") + } + } else { + // just remove the one arg for now + edge.Args = newArgs + } + + // always free the database entry + if err := obj.FreeEdge(f1, f2, arg); err != nil { + return err + } + } + } + } + + // Now check the vertices... + vs := []interfaces.Func{} + for vertex, count := range obj.vertices { + if count != 0 { + continue + } + + // safety check, vertex is still in use by an edge + for x := range obj.edges { + if x.f1 == vertex || x.f2 == vertex { + // programming error + return fmt.Errorf("vertex unexpectedly still in use: %p %s", vertex, vertex) + } + } + + vs = append(vs, vertex) + } + + for _, vertex := range vs { + if err := graphAPI.DeleteVertex(vertex); err != nil { + return errwrap.Wrapf(err, "vertex deletion error") + } + // free the database entry + if err := obj.FreeVertex(vertex); err != nil { + return err + } + } + + return nil +} + +// makeEdge looks up an edge with the "hash" input we are seeking. If it doesn't +// find a match, it returns a new one with those fields. +func (obj *RefCount) makeEdge(f1, f2 interfaces.Func, arg string) *RefCountEdge { + for k := range obj.edges { + //if k == nil { // programming error + // continue + //} + if k.f1 == f1 && k.f2 == f2 && k.arg == arg { + return k + } + } + return &RefCountEdge{ // not found, so make a new one! + f1: f1, + f2: f2, + arg: arg, + } +} diff --git a/lang/funcs/dage/txn.go b/lang/funcs/dage/txn.go new file mode 100644 index 0000000000..41a669cce6 --- /dev/null +++ b/lang/funcs/dage/txn.go @@ -0,0 +1,626 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Package dage implements a DAG function engine. +// TODO: can we rename this to something more interesting? +package dage + +import ( + "fmt" + "sort" + "sync" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/pgraph" +) + +// PostReverseCommit specifies that if we run Reverse, and we had previous items +// pending for Commit, that we should Commit them after our Reverse runs. +// Otherwise they remain on the pending queue and wait for you to run Commit. +const PostReverseCommit = false + +// GraphvizDebug enables writing graphviz graphs on each commit. This is very +// slow. +const GraphvizDebug = false + +// opapi is the input for any op. This allows us to keeps things compact and it +// also allows us to change API slightly without re-writing code. +type opapi struct { + GraphAPI interfaces.GraphAPI + RefCount *RefCount +} + +// opfn is an interface that holds the normal op, and the reverse op if we need +// to rollback from the forward fn. Implementations of each op can decide to +// store some internal state when running the forward op which might be needed +// for the possible future reverse op. +type opfn interface { + fmt.Stringer + + Fn(*opapi) error + Rev(*opapi) error +} + +type opfnSkipRev interface { + opfn + + // Skip tells us if this op should be skipped from reversing. + Skip() bool + + // SetSkip specifies that this op should be skipped from reversing. + SetSkip(bool) +} + +type opfnFlag interface { + opfn + + // Flag reads some misc data. + Flag() interface{} + + // SetFlag sets some misc data. + SetFlag(interface{}) +} + +// revOp returns the reversed op from an op by packing or unpacking it. +func revOp(op opfn) opfn { + if skipOp, ok := op.(opfnSkipRev); ok && skipOp.Skip() { + return nil // skip + } + + // XXX: is the reverse of a reverse just undoing it? maybe not but might not matter for us + if newOp, ok := op.(*opRev); ok { + + if newFlagOp, ok := op.(opfnFlag); ok { + newFlagOp.SetFlag("does this rev of rev even happen?") + } + + return newOp.Op // unpack it + } + + return &opRev{ + Op: op, + + opFlag: &opFlag{}, + } // pack it +} + +// opRev switches the Fn and Rev methods by wrapping the contained op in each +// other. +type opRev struct { + Op opfn + + *opFlag +} + +func (obj *opRev) Fn(opapi *opapi) error { + return obj.Op.Rev(opapi) +} + +func (obj *opRev) Rev(opapi *opapi) error { + return obj.Op.Fn(opapi) +} + +func (obj *opRev) String() string { + return "rev(" + obj.Op.String() + ")" // TODO: is this correct? +} + +type opSkip struct { + skip bool +} + +func (obj *opSkip) Skip() bool { + return obj.skip +} + +func (obj *opSkip) SetSkip(skip bool) { + obj.skip = skip +} + +type opFlag struct { + flag interface{} +} + +func (obj *opFlag) Flag() interface{} { + return obj.flag +} + +func (obj *opFlag) SetFlag(flag interface{}) { + obj.flag = flag +} + +type opAddVertex struct { + F interfaces.Func + + *opSkip + *opFlag +} + +func (obj *opAddVertex) Fn(opapi *opapi) error { + if opapi.RefCount.VertexInc(obj.F) { + // add if we're the first reference + return opapi.GraphAPI.AddVertex(obj.F) + } + + return nil +} + +func (obj *opAddVertex) Rev(opapi *opapi) error { + opapi.RefCount.VertexDec(obj.F) + // any removal happens in gc + return nil +} + +func (obj *opAddVertex) String() string { + return fmt.Sprintf("AddVertex: %+v", obj.F) +} + +type opAddEdge struct { + F1 interfaces.Func + F2 interfaces.Func + FE *interfaces.FuncEdge + + *opSkip + *opFlag +} + +func (obj *opAddEdge) Fn(opapi *opapi) error { + if obj.F1 == obj.F2 { // simplify below code/logic with this easy check + return fmt.Errorf("duplicate vertex cycle") + } + + opapi.RefCount.EdgeInc(obj.F1, obj.F2, obj.FE) + + fe := obj.FE // squish multiple edges together if one already exists + if edge := opapi.GraphAPI.FindEdge(obj.F1, obj.F2); edge != nil { + args := make(map[string]struct{}) + for _, x := range obj.FE.Args { + args[x] = struct{}{} + } + for _, x := range edge.Args { + args[x] = struct{}{} + } + if len(args) != len(obj.FE.Args)+len(edge.Args) { + // programming error + return fmt.Errorf("duplicate arg found") + } + newArgs := []string{} + for x := range args { + newArgs = append(newArgs, x) + } + sort.Strings(newArgs) // for consistency? + fe = &interfaces.FuncEdge{ + Args: newArgs, + } + } + + // The dage API currently smooshes together any existing edge args with + // our new edge arg names. It also adds the vertices if needed. + if err := opapi.GraphAPI.AddEdge(obj.F1, obj.F2, fe); err != nil { + return err + } + + return nil +} + +func (obj *opAddEdge) Rev(opapi *opapi) error { + opapi.RefCount.EdgeDec(obj.F1, obj.F2, obj.FE) + return nil +} + +func (obj *opAddEdge) String() string { + return fmt.Sprintf("AddEdge: %+v -> %+v (%+v)", obj.F1, obj.F2, obj.FE) +} + +type opDeleteVertex struct { + F interfaces.Func + + *opSkip + *opFlag +} + +func (obj *opDeleteVertex) Fn(opapi *opapi) error { + if opapi.RefCount.VertexDec(obj.F) { + //delete(opapi.RefCount.Vertices, obj.F) // don't GC this one + if err := opapi.RefCount.FreeVertex(obj.F); err != nil { + panic("could not free vertex") + } + return opapi.GraphAPI.DeleteVertex(obj.F) // do it here instead + } + return nil +} + +func (obj *opDeleteVertex) Rev(opapi *opapi) error { + if opapi.RefCount.VertexInc(obj.F) { + return opapi.GraphAPI.AddVertex(obj.F) + } + return nil +} + +func (obj *opDeleteVertex) String() string { + return fmt.Sprintf("DeleteVertex: %+v", obj.F) +} + +// graphTxn holds the state of a transaction and runs it when needed. When this +// has been setup and initialized, it implements the Txn API that can be used by +// functions in their Stream method to modify the function graph while it is +// "running". +type graphTxn struct { + // Lock is a handle to the lock function to call before the operation. + Lock func() + + // Unlock is a handle to the unlock function to call before the + // operation. + Unlock func() + + // GraphAPI is a handle pointing to the graph API implementation we're + // using for any txn operations. + GraphAPI interfaces.GraphAPI + + // RefCount keeps track of vertex and edge references across the entire + // graph. + RefCount *RefCount + + // FreeFunc is a function that will get called by a well-behaved user + // when we're done with this Txn. + FreeFunc func() + + // ops is a list of operations to run on a graph + ops []opfn + + // rev is a list of reverse operations to run on a graph + rev []opfn + + // mutex guards changes to the ops list + mutex *sync.Mutex +} + +// init must be called to initialized the struct before first use. This is +// private because the creator, not the user should run it. +func (obj *graphTxn) init() interfaces.Txn { + obj.ops = []opfn{} + obj.rev = []opfn{} + obj.mutex = &sync.Mutex{} + + return obj // return self so it can be called in a chain +} + +// Copy returns a new child Txn that has the same handles, but a separate state. +// This allows you to do an Add*/Commit/Reverse that isn't affected by a +// different user of this transaction. +// TODO: FreeFunc isn't well supported here. Replace or remove this entirely? +func (obj *graphTxn) Copy() interfaces.Txn { + txn := &graphTxn{ + Lock: obj.Lock, + Unlock: obj.Unlock, + GraphAPI: obj.GraphAPI, + RefCount: obj.RefCount, // this is shared across all txn's + // FreeFunc is shared with the parent. + } + return txn.init() +} + +// AddVertex adds a vertex to the running graph. The operation will get +// completed when Commit is run. +// XXX: should this be pgraph.Vertex instead of interfaces.Func ? +func (obj *graphTxn) AddVertex(f interfaces.Func) interfaces.Txn { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + opfn := &opAddVertex{ + F: f, + + opSkip: &opSkip{}, + opFlag: &opFlag{}, + } + obj.ops = append(obj.ops, opfn) + + return obj // return self so it can be called in a chain +} + +// AddEdge adds an edge to the running graph. The operation will get completed +// when Commit is run. +// XXX: should this be pgraph.Vertex instead of interfaces.Func ? +// XXX: should this be pgraph.Edge instead of *interfaces.FuncEdge ? +func (obj *graphTxn) AddEdge(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) interfaces.Txn { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + opfn := &opAddEdge{ + F1: f1, + F2: f2, + FE: fe, + + opSkip: &opSkip{}, + opFlag: &opFlag{}, + } + obj.ops = append(obj.ops, opfn) + + // NOTE: we can't build obj.rev yet because in this case, we'd need to + // know if the runtime graph contained one of the two pre-existing + // vertices or not, or if it would get added implicitly by this op! + + return obj // return self so it can be called in a chain +} + +// DeleteVertex adds a vertex to the running graph. The operation will get +// completed when Commit is run. +// XXX: should this be pgraph.Vertex instead of interfaces.Func ? +func (obj *graphTxn) DeleteVertex(f interfaces.Func) interfaces.Txn { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + opfn := &opDeleteVertex{ + F: f, + + opSkip: &opSkip{}, + opFlag: &opFlag{}, + } + obj.ops = append(obj.ops, opfn) + + return obj // return self so it can be called in a chain +} + +// AddGraph adds a graph to the running graph. The operation will get completed +// when Commit is run. This function panics if your graph contains vertices that +// are not of type interfaces.Func or if your edges are not of type +// *interfaces.FuncEdge. +func (obj *graphTxn) AddGraph(g *pgraph.Graph) interfaces.Txn { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + for _, v := range g.Vertices() { + f, ok := v.(interfaces.Func) + if !ok { + panic("not a Func") + } + //obj.AddVertex(f) // easy + opfn := &opAddVertex{ // replicate AddVertex + F: f, + + opSkip: &opSkip{}, + opFlag: &opFlag{}, + } + obj.ops = append(obj.ops, opfn) + } + + for v1, m := range g.Adjacency() { + f1, ok := v1.(interfaces.Func) + if !ok { + panic("not a Func") + } + for v2, e := range m { + f2, ok := v2.(interfaces.Func) + if !ok { + panic("not a Func") + } + fe, ok := e.(*interfaces.FuncEdge) + if !ok { + panic("not a *FuncEdge") + } + + //obj.AddEdge(f1, f2, fe) // easy + opfn := &opAddEdge{ // replicate AddEdge + F1: f1, + F2: f2, + FE: fe, + + opSkip: &opSkip{}, + opFlag: &opFlag{}, + } + obj.ops = append(obj.ops, opfn) + } + } + + return obj // return self so it can be called in a chain +} + +// commit runs the pending transaction. This is the lockless version that is +// only used internally. +func (obj *graphTxn) commit() error { + if len(obj.ops) == 0 { // nothing to do + return nil + } + + // TODO: Instead of requesting the below locks, it's conceivable that we + // could either write an engine that doesn't require pausing the graph + // with a lock, or one that doesn't in the specific case being changed + // here need locks. And then in theory we'd have improved performance + // from the function engine. For our function consumers, the Txn API + // would never need to change, so we don't break API! A simple example + // is the len(ops) == 0 one right above. A simplification, but shows we + // aren't forced to call the locks even when we get Commit called here. + + // Now request the lock from the actual graph engine. + obj.Lock() + defer obj.Unlock() + + // Now request the ref count mutex. This may seem redundant, but it's + // not. The above graph engine Lock might allow more than one commit + // through simultaneously depending on implementation. The actual count + // mathematics must not, and so it has a separate lock. We could lock it + // per-operation, but that would probably be a lot slower. + obj.RefCount.Lock() + defer obj.RefCount.Unlock() + + // TODO: we don't need to do this anymore, because the engine does it! + // Copy the graph structure, perform the ops, check we didn't add a + // cycle, and if it's safe, do the real thing. Otherwise error here. + //g := obj.Graph.Copy() // copy the graph structure + //for _, x := range obj.ops { + // x(g) // call it + //} + //if _, err := g.TopologicalSort(); err != nil { + // return errwrap.Wrapf(err, "topo sort failed in txn commit") + //} + // FIXME: is there anything else we should check? Should we type-check? + + // Now do it for real... + obj.rev = []opfn{} // clear it for safety + opapi := &opapi{ + GraphAPI: obj.GraphAPI, + RefCount: obj.RefCount, + } + for _, op := range obj.ops { + if err := op.Fn(opapi); err != nil { // call it + // something went wrong (we made a cycle?) + obj.rev = []opfn{} // clear it, we didn't succeed + return err + } + + op = revOp(op) // reverse the op! + if op != nil { + obj.rev = append(obj.rev, op) // add the reverse op + //obj.rev = append([]opfn{op}, obj.rev...) // add to front + } + } + obj.ops = []opfn{} // clear it + + // garbage collect anything that hit zero! + // XXX: add gc function to this struct and pass in opapi instead? + if err := obj.RefCount.GC(obj.GraphAPI); err != nil { + // programming error or ghosts + return err + } + + // XXX: running this on each commit has a huge performance hit. + // XXX: we could write out the .dot files and run graphviz afterwards + if engine, ok := obj.GraphAPI.(*Engine); ok && GraphvizDebug { + //d := time.Now().Unix() + //if err := engine.graph.ExecGraphviz(fmt.Sprintf("/tmp/txn-graphviz-%d.dot", d)); err != nil { + // panic("no graphviz") + //} + if err := engine.Graphviz(""); err != nil { + panic(err) // XXX: improve me + } + + //gv := &pgraph.Graphviz{ + // Filename: fmt.Sprintf("/tmp/txn-graphviz-%d.dot", d), + // Graphs: map[*pgraph.Graph]*pgraph.GraphvizOpts{ + // engine.graph: nil, + // }, + //} + //if err := gv.Exec(); err != nil { + // panic("no graphviz") + //} + } + return nil +} + +// Commit runs the pending transaction. If there was a pending reverse +// transaction that could have run (it would have been available following a +// Commit success) then this will erase that transaction. Usually you run cycles +// of Commit, followed by Reverse, or only Commit. (You obviously have to +// populate operations before the Commit is run.) +func (obj *graphTxn) Commit() error { + // Lock our internal state mutex first... this prevents other AddVertex + // or similar calls from interferring with our work here. + obj.mutex.Lock() + defer obj.mutex.Unlock() + + return obj.commit() +} + +// Clear erases any pending transactions that weren't committed yet. +func (obj *graphTxn) Clear() { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + obj.ops = []opfn{} // clear it +} + +// Reverse is like Commit, but it commits the reverse transaction to the one +// that previously ran with Commit. If the PostReverseCommit global has been set +// then if there were pending commit operations when this was run, then they are +// run at the end of a successful Reverse. It is generally recommended to not +// queue any operations for Commit if you plan on doing a Reverse, or to run a +// Clear before running Reverse if you want to discard the pending commits. +func (obj *graphTxn) Reverse() error { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + // first commit all the rev stuff... and then run the pending ops... + + ops := []opfn{} // save a copy + for _, op := range obj.ops { // copy + ops = append(ops, op) + } + obj.ops = []opfn{} // clear + + //for _, op := range obj.rev + for i := len(obj.rev) - 1; i >= 0; i-- { // copy in the rev stuff to commit! + op := obj.rev[i] + // mark these as being not reversible (so skip them on reverse!) + if skipOp, ok := op.(opfnSkipRev); ok { + skipOp.SetSkip(true) + } + obj.ops = append(obj.ops, op) + } + + //rev := []func(interfaces.GraphAPI){} // for the copy + //for _, op := range obj.rev { // copy + // rev = append(rev, op) + //} + obj.rev = []opfn{} // clear + + //rollback := func() { + // //for _, op := range rev { // from our safer copy + // //for _, op := range obj.ops { // copy back out the rev stuff + // for i := len(obj.ops) - 1; i >= 0; i-- { // copy in the rev stuff to commit! + // op := obj.rev[i] + // obj.rev = append(obj.rev, op) + // } + // obj.ops = []opfn{} // clear + // for _, op := range ops { // copy the original ops back in + // obj.ops = append(obj.ops, op) + // } + //} + // first commit the reverse stuff + if err := obj.commit(); err != nil { // lockless version + // restore obj.rev and obj.ops + //rollback() // probably not needed + return err + } + + // then if we had normal ops queued up, run those or at least restore... + for _, op := range ops { // copy + obj.ops = append(obj.ops, op) + } + + if PostReverseCommit { + return obj.commit() // lockless version + } + + return nil +} + +// Erase removes the historical information that Reverse would run after Commit. +func (obj *graphTxn) Erase() { + obj.mutex.Lock() + defer obj.mutex.Unlock() + + obj.rev = []opfn{} // clear it +} + +// Free releases the wait group that was used to lock around this Txn if needed. +// It should get called when we're done with any Txn. +// TODO: this is only used for the initial Txn. Consider expanding it's use. We +// might need to allow Clear to call it as part of the clearing. +func (obj *graphTxn) Free() { + if obj.FreeFunc != nil { + obj.FreeFunc() + } +} diff --git a/lang/funcs/dage/txn_test.go b/lang/funcs/dage/txn_test.go new file mode 100644 index 0000000000..d615eead21 --- /dev/null +++ b/lang/funcs/dage/txn_test.go @@ -0,0 +1,503 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//go:build !root + +package dage + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/pgraph" + "github.com/purpleidea/mgmt/util" +) + +type testGraphAPI struct { + graph *pgraph.Graph +} + +func (obj *testGraphAPI) AddVertex(f interfaces.Func) error { + v, ok := f.(pgraph.Vertex) + if !ok { + return fmt.Errorf("can't use func as vertex") + } + obj.graph.AddVertex(v) + return nil +} +func (obj *testGraphAPI) AddEdge(f1, f2 interfaces.Func, fe *interfaces.FuncEdge) error { + v1, ok := f1.(pgraph.Vertex) + if !ok { + return fmt.Errorf("can't use func as vertex") + } + v2, ok := f2.(pgraph.Vertex) + if !ok { + return fmt.Errorf("can't use func as vertex") + } + obj.graph.AddEdge(v1, v2, fe) + return nil +} + +func (obj *testGraphAPI) DeleteVertex(f interfaces.Func) error { + v, ok := f.(pgraph.Vertex) + if !ok { + return fmt.Errorf("can't use func as vertex") + } + obj.graph.DeleteVertex(v) + return nil +} + +func (obj *testGraphAPI) DeleteEdge(fe *interfaces.FuncEdge) error { + obj.graph.DeleteEdge(fe) + return nil +} + +//func (obj *testGraphAPI) AddGraph(*pgraph.Graph) error { +// return fmt.Errorf("not implemented") +//} + +//func (obj *testGraphAPI) Adjacency() map[interfaces.Func]map[interfaces.Func]*interfaces.FuncEdge { +// panic("not implemented") +//} + +func (obj *testGraphAPI) HasVertex(f interfaces.Func) bool { + v, ok := f.(pgraph.Vertex) + if !ok { + panic("can't use func as vertex") + } + return obj.graph.HasVertex(v) +} + +func (obj *testGraphAPI) LookupEdge(fe *interfaces.FuncEdge) (interfaces.Func, interfaces.Func, bool) { + v1, v2, b := obj.graph.LookupEdge(fe) + if !b { + return nil, nil, b + } + + f1, ok := v1.(interfaces.Func) + if !ok { + panic("can't use vertex as func") + } + f2, ok := v2.(interfaces.Func) + if !ok { + panic("can't use vertex as func") + } + return f1, f2, true +} + +func (obj *testGraphAPI) FindEdge(f1, f2 interfaces.Func) *interfaces.FuncEdge { + edge := obj.graph.FindEdge(f1, f2) + if edge == nil { + return nil + } + fe, ok := edge.(*interfaces.FuncEdge) + if !ok { + panic("edge is not a FuncEdge") + } + + return fe +} + +type testNullFunc struct { + name string +} + +func (obj *testNullFunc) String() string { return obj.name } +func (obj *testNullFunc) Info() *interfaces.Info { return nil } +func (obj *testNullFunc) Validate() error { return nil } +func (obj *testNullFunc) Init(*interfaces.Init) error { return nil } +func (obj *testNullFunc) Stream(context.Context) error { return nil } + +func TestTxn1(t *testing.T) { + graph, err := pgraph.NewGraph("test") + if err != nil { + t.Errorf("err: %+v", err) + return + } + testGraphAPI := &testGraphAPI{graph: graph} + mutex := &sync.Mutex{} + + graphTxn := &graphTxn{ + GraphAPI: testGraphAPI, + Lock: mutex.Lock, + Unlock: mutex.Unlock, + RefCount: (&RefCount{}).Init(), + } + txn := graphTxn.init() + + f1 := &testNullFunc{"f1"} + + if err := txn.AddVertex(f1).Commit(); err != nil { + t.Errorf("commit err: %+v", err) + return + } + + if l, i := len(graph.Adjacency()), 1; l != i { + t.Errorf("got len of: %d", l) + t.Errorf("exp len of: %d", i) + return + } + + if err := txn.Reverse(); err != nil { + t.Errorf("reverse err: %+v", err) + return + } + + if l, i := len(graph.Adjacency()), 0; l != i { + t.Errorf("got len of: %d", l) + t.Errorf("exp len of: %d", i) + return + } +} + +type txnTestOp func(*pgraph.Graph, interfaces.Txn) error + +func TestTxnTable(t *testing.T) { + + type test struct { // an individual test + name string + actions []txnTestOp + } + testCases := []test{} + { + f1 := &testNullFunc{"f1"} + + testCases = append(testCases, test{ + name: "simple add vertex", + actions: []txnTestOp{ + //func(g *pgraph.Graph, txn interfaces.Txn) error { + // txn.AddVertex(f1) + // return nil + //}, + //func(g *pgraph.Graph, txn interfaces.Txn) error { + // return txn.Commit() + //}, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddVertex(f1).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 1; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Reverse() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 0; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + }, + }) + } + { + f1 := &testNullFunc{"f1"} + f2 := &testNullFunc{"f2"} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "simple add edge", + actions: []txnTestOp{ + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddEdge(f1, f2, e1).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 1; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Reverse() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 0; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 0; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + }, + }) + } + { + f1 := &testNullFunc{"f1"} + f2 := &testNullFunc{"f2"} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "simple add edge two-step", + actions: []txnTestOp{ + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddVertex(f1).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddEdge(f1, f2, e1).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 1; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + // Reverse only undoes what happened since the + // previous commit, so only one of the nodes is + // left at the end. + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Reverse() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 1; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 0; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + }, + }) + } + { + f1 := &testNullFunc{"f1"} + f2 := &testNullFunc{"f2"} + e1 := testEdge("e1") + + testCases = append(testCases, test{ + name: "simple two add edge, reverse", + actions: []txnTestOp{ + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddVertex(f1).AddVertex(f2).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.AddEdge(f1, f2, e1).Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 1; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + // Reverse only undoes what happened since the + // previous commit, so only one of the nodes is + // left at the end. + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Reverse() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 0; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + }, + }) + } + { + f1 := &testNullFunc{"f1"} + f2 := &testNullFunc{"f2"} + f3 := &testNullFunc{"f3"} + f4 := &testNullFunc{"f4"} + e1 := testEdge("e1") + e2 := testEdge("e2") + e3 := testEdge("e3") + e4 := testEdge("e4") + + testCases = append(testCases, test{ + name: "simple add/delete", + actions: []txnTestOp{ + func(g *pgraph.Graph, txn interfaces.Txn) error { + txn.AddVertex(f1).AddEdge(f1, f2, e1) + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + txn.AddVertex(f1).AddEdge(f1, f3, e2) + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 3; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + txn.AddEdge(f2, f4, e3) + txn.AddEdge(f3, f4, e4) + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Commit() + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 4; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 4; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + // debug + //func(g *pgraph.Graph, txn interfaces.Txn) error { + // fileName := "/tmp/graphviz-txn1.dot" + // if err := g.ExecGraphviz(fileName); err != nil { + // return fmt.Errorf("writing graph failed at: %s, err: %+v", fileName, err) + // } + // return nil + //}, + func(g *pgraph.Graph, txn interfaces.Txn) error { + return txn.Reverse() + }, + // debug + //func(g *pgraph.Graph, txn interfaces.Txn) error { + // fileName := "/tmp/graphviz-txn2.dot" + // if err := g.ExecGraphviz(fileName); err != nil { + // return fmt.Errorf("writing graph failed at: %s, err: %+v", fileName, err) + // } + // return nil + //}, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Adjacency()), 3; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + func(g *pgraph.Graph, txn interfaces.Txn) error { + if l, i := len(g.Edges()), 2; l != i { + return fmt.Errorf("got len of: %d, exp len of: %d", l, i) + } + return nil + }, + }, + }) + } + if testing.Short() { + t.Logf("available tests:") + } + names := []string{} + for index, tc := range testCases { // run all the tests + if tc.name == "" { + t.Errorf("test #%d: not named", index) + continue + } + if util.StrInList(tc.name, names) { + t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) + continue + } + names = append(names, tc.name) + + //if index != 3 { // hack to run a subset (useful for debugging) + //if tc.name != "simple txn" { + // continue + //} + + testName := fmt.Sprintf("test #%d (%s)", index, tc.name) + if testing.Short() { // make listing tests easier + t.Logf("%s", testName) + continue + } + t.Run(testName, func(t *testing.T) { + name, actions := tc.name, tc.actions + + t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) + + //logf := func(format string, v ...interface{}) { + // t.Logf(fmt.Sprintf("test #%d", index)+": "+format, v...) + //} + + graph, err := pgraph.NewGraph("test") + if err != nil { + t.Errorf("err: %+v", err) + return + } + testGraphAPI := &testGraphAPI{graph: graph} + mutex := &sync.Mutex{} + + graphTxn := &graphTxn{ + GraphAPI: testGraphAPI, + Lock: mutex.Lock, + Unlock: mutex.Unlock, + RefCount: (&RefCount{}).Init(), + } + txn := graphTxn.init() + + // Run a list of actions, passing the returned txn (if + // any) to the next action. Any error kills it all. + for i, action := range actions { + if err := action(graph, txn); err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: action #%d failed with: %+v", index, i, err) + return + } + } + }) + } + + if testing.Short() { + t.Skip("skipping all tests...") + } +} diff --git a/lang/funcs/dage/util_test.go b/lang/funcs/dage/util_test.go new file mode 100644 index 0000000000..b241f9b894 --- /dev/null +++ b/lang/funcs/dage/util_test.go @@ -0,0 +1,30 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//go:build !root + +package dage + +import ( + "github.com/purpleidea/mgmt/lang/interfaces" +) + +func testEdge(name string) *interfaces.FuncEdge { + return &interfaces.FuncEdge{ + Args: []string{name}, + } +} diff --git a/lang/funcs/engine.go b/lang/funcs/engine.go deleted file mode 100644 index c8819e128e..0000000000 --- a/lang/funcs/engine.go +++ /dev/null @@ -1,697 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2023+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package funcs - -import ( - "context" - "fmt" - "sync" - - "github.com/purpleidea/mgmt/engine" - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - "github.com/purpleidea/mgmt/pgraph" - "github.com/purpleidea/mgmt/util/errwrap" -) - -// State represents the state of a function vertex. This corresponds to an AST -// expr, which is the memory address (pointer) in the graph. -type State struct { - Expr interfaces.Expr // pointer to the expr vertex - - handle interfaces.Func // the function (if not nil, we've found it on init) - - ctx context.Context // used for shutting down each Stream function. - cancel context.CancelFunc - - init bool // have we run Init on our func? - ready bool // has it received all the args it needs at least once? - loaded bool // has the func run at least once ? - closed bool // did we close ourself down? - - notify chan struct{} // ping here when new input values exist - - input chan types.Value // the top level type must be a struct - output chan types.Value - - mutex *sync.RWMutex // concurrency guard for modifying Expr with String/SetValue -} - -// Init creates the function state if it can be found in the registered list. -func (obj *State) Init() error { - handle, err := obj.Expr.Func() // build one and store it, don't re-gen - if err != nil { - return err - } - if err := handle.Validate(); err != nil { - return errwrap.Wrapf(err, "could not validate func") - } - obj.handle = handle - - sig := obj.handle.Info().Sig - if sig.Kind != types.KindFunc { - return fmt.Errorf("must be kind func") - } - if len(sig.Ord) > 0 { - // since we accept input, better get our notification chan built - obj.notify = make(chan struct{}) - } - - obj.input = make(chan types.Value) // we close this when we're done - obj.output = make(chan types.Value) // we create it, func closes it - - obj.mutex = &sync.RWMutex{} - - return nil -} - -// String satisfies fmt.Stringer so that these print nicely. -func (obj *State) String() string { - // TODO: use global mutex since it's harder to add state specific mutex - //obj.mutex.RLock() // prevent race detector issues against SetValue - //defer obj.mutex.RUnlock() - // FIXME: also add read locks on any of the children Expr in obj.Expr - return obj.Expr.String() -} - -// Engine represents the running time varying directed acyclic function graph. -type Engine struct { - Graph *pgraph.Graph - Hostname string - World engine.World - Debug bool - Logf func(format string, v ...interface{}) - - // Glitch: https://en.wikipedia.org/wiki/Reactive_programming#Glitches - Glitch bool // allow glitching? (more responsive, but less accurate) - - ag chan error // used to aggregate fact events without reflect - agLock *sync.Mutex - agCount int // last one turns out the light (closes the ag channel) - - topologicalSort []pgraph.Vertex // cached sorting of the graph for perf - - state map[pgraph.Vertex]*State // state associated with the vertex - - mutex *sync.RWMutex // concurrency guard for the table map - table map[pgraph.Vertex]types.Value // live table of output values - - loaded bool // are all of the funcs loaded? - loadedChan chan struct{} // funcs loaded signal - - streamChan chan error // signals a new graph can be created or problem - - ctx context.Context // used for shutting down each Stream function. - cancel context.CancelFunc - - closeChan chan struct{} // close signal - wg *sync.WaitGroup -} - -// Init initializes the struct. This is the first call you must make. Do not -// proceed with calls to other methods unless this succeeds first. This also -// loads all the functions by calling Init on each one in the graph. -// TODO: should Init take the graph as an input arg to keep it as a private -// field? -func (obj *Engine) Init() error { - obj.ag = make(chan error) - obj.agLock = &sync.Mutex{} - obj.state = make(map[pgraph.Vertex]*State) - obj.mutex = &sync.RWMutex{} - obj.table = make(map[pgraph.Vertex]types.Value) - obj.loadedChan = make(chan struct{}) - obj.streamChan = make(chan error) - obj.closeChan = make(chan struct{}) - obj.wg = &sync.WaitGroup{} - topologicalSort, err := obj.Graph.TopologicalSort() - if err != nil { - return errwrap.Wrapf(err, "topo sort failed") - } - obj.topologicalSort = topologicalSort // cache the result - - obj.ctx, obj.cancel = context.WithCancel(context.Background()) // top - - for _, vertex := range obj.Graph.Vertices() { - // is this an interface we can use? - if _, exists := obj.state[vertex]; exists { - return fmt.Errorf("vertex (%+v) is not unique in the graph", vertex) - } - - expr, ok := vertex.(interfaces.Expr) - if !ok { - return fmt.Errorf("vertex (%+v) was not an expr", vertex) - } - - if obj.Debug { - obj.Logf("Loading func `%s`", vertex) - } - - innerCtx, innerCancel := context.WithCancel(obj.ctx) - obj.state[vertex] = &State{ - Expr: expr, - - ctx: innerCtx, - cancel: innerCancel, - } // store some state! - - e1 := obj.state[vertex].Init() - e2 := errwrap.Wrapf(e1, "error loading func `%s`", vertex) - err = errwrap.Append(err, e2) // list of errors - } - if err != nil { // usually due to `not found` errors - return errwrap.Wrapf(err, "could not load requested funcs") - } - return nil -} - -// Validate the graph type checks properly and other tests. Must run Init first. -// This should check that: (1) all vertices have the correct number of inputs, -// (2) that the *Info signatures all match correctly, (3) that the argument -// names match correctly, and that the whole graph is statically correct. -func (obj *Engine) Validate() error { - inList := func(needle interfaces.Func, haystack []interfaces.Func) bool { - if needle == nil { - panic("nil value passed to inList") // catch bugs! - } - for _, x := range haystack { - if needle == x { - return true - } - } - return false - } - var err error - ptrs := []interfaces.Func{} // Func is a ptr - for _, vertex := range obj.Graph.Vertices() { - node := obj.state[vertex] - // TODO: this doesn't work for facts because they're in the Func - // duplicate pointers would get closed twice, causing a panic... - if inList(node.handle, ptrs) { // check for duplicate ptrs! - e := fmt.Errorf("vertex `%s` has duplicate ptr", vertex) - err = errwrap.Append(err, e) - } - ptrs = append(ptrs, node.handle) - } - for _, edge := range obj.Graph.Edges() { - if _, ok := edge.(*interfaces.FuncEdge); !ok { - e := fmt.Errorf("edge `%s` was not the correct type", edge) - err = errwrap.Append(err, e) - } - } - if err != nil { - return err // stage the errors so the user can fix many at once! - } - - // check if vertices expecting inputs have them - for vertex, count := range obj.Graph.InDegree() { - node := obj.state[vertex] - if exp := len(node.handle.Info().Sig.Ord); exp != count { - e := fmt.Errorf("expected %d inputs to `%s`, got %d", exp, node, count) - if obj.Debug { - obj.Logf("expected %d inputs to `%s`, got %d", exp, node, count) - obj.Logf("expected: %+v for `%s`", node.handle.Info().Sig.Ord, node) - } - err = errwrap.Append(err, e) - } - } - - // expected vertex -> argName - expected := make(map[*State]map[string]int) // expected input fields - for vertex1 := range obj.Graph.Adjacency() { - // check for outputs that don't go anywhere? - //node1 := obj.state[vertex1] - //if len(obj.Graph.Adjacency()[vertex1]) == 0 { // no vertex1 -> vertex2 - // if node1.handle.Info().Sig.Output != nil { - // // an output value goes nowhere... - // } - //} - for vertex2 := range obj.Graph.Adjacency()[vertex1] { // populate - node2 := obj.state[vertex2] - expected[node2] = make(map[string]int) - for _, key := range node2.handle.Info().Sig.Ord { - expected[node2][key] = 1 - } - } - } - - for vertex1 := range obj.Graph.Adjacency() { - node1 := obj.state[vertex1] - for vertex2, edge := range obj.Graph.Adjacency()[vertex1] { - node2 := obj.state[vertex2] - edge := edge.(*interfaces.FuncEdge) - // check vertex1 -> vertex2 (with e) is valid - - for _, arg := range edge.Args { // loop over each arg - sig := node2.handle.Info().Sig - if len(sig.Ord) == 0 { - e := fmt.Errorf("no input expected from `%s` to `%s` with arg `%s`", node1, node2, arg) - err = errwrap.Append(err, e) - continue - } - - if count, exists := expected[node2][arg]; !exists { - e := fmt.Errorf("wrong input name from `%s` to `%s` with arg `%s`", node1, node2, arg) - err = errwrap.Append(err, e) - } else if count == 0 { - e := fmt.Errorf("duplicate input from `%s` to `%s` with arg `%s`", node1, node2, arg) - err = errwrap.Append(err, e) - } - expected[node2][arg]-- // subtract one use - - out := node1.handle.Info().Sig.Out - if out == nil { - e := fmt.Errorf("no output possible from `%s` to `%s` with arg `%s`", node1, node2, arg) - err = errwrap.Append(err, e) - continue - } - typ, exists := sig.Map[arg] // key in struct - if !exists { - // second check of this! - e := fmt.Errorf("wrong input name from `%s` to `%s` with arg `%s`", node1, node2, arg) - err = errwrap.Append(err, errwrap.Wrapf(e, "programming error")) - continue - } - - if typ.Kind == types.KindVariant { // FIXME: hack for now - // pass (input arg variants) - } else if out.Kind == types.KindVariant { // FIXME: hack for now - // pass (output arg variants) - } else if typ.Cmp(out) != nil { - e := fmt.Errorf("type mismatch from `%s` (%s) to `%s` (%s) with arg `%s`", node1, out, node2, typ, arg) - err = errwrap.Append(err, e) - } - } - } - } - - // check for leftover function inputs which weren't filled up by outputs - // (we're trying to call a function with fewer input args than required) - for node, m := range expected { // map[*State]map[string]int - for arg, count := range m { - if count != 0 { // count should be zero if all were used - e := fmt.Errorf("missing input to `%s` on arg `%s`", node, arg) - err = errwrap.Append(err, e) - } - } - } - - if err != nil { - return err // stage the errors so the user can fix many at once! - } - - return nil -} - -// Run starts up this function engine and gets it all running. It errors if the -// startup failed for some reason. On success, use the Stream and Table methods -// for future interaction with the engine, and the Close method to shut it off. -func (obj *Engine) Run() error { - if len(obj.topologicalSort) == 0 { // no funcs to load! - close(obj.loadedChan) - close(obj.streamChan) - return nil - } - - // TODO: build a timer that runs while we wait for all funcs to startup. - // after some delay print a message to tell us which funcs we're waiting - // for to startup and that they are slow and blocking everyone, and then - // fail permanently after the timeout so that bad code can't block this! - - // loop through all funcs that we might need - obj.agAdd(len(obj.topologicalSort)) - for _, vertex := range obj.topologicalSort { - node := obj.state[vertex] - if obj.Debug { - obj.SafeLogf("Startup func `%s`", node) - } - - incoming := obj.Graph.IncomingGraphVertices(vertex) // []Vertex - - init := &interfaces.Init{ - Hostname: obj.Hostname, - Input: node.input, - Output: node.output, - World: obj.World, - Debug: obj.Debug, - Logf: func(format string, v ...interface{}) { - obj.Logf("func: "+format, v...) - }, - } - if err := node.handle.Init(init); err != nil { - return errwrap.Wrapf(err, "could not init func `%s`", node) - } - node.init = true // we've successfully initialized - - // no incoming edges, so no incoming data - if len(incoming) == 0 { // TODO: do this here or earlier? - close(node.input) - } else { - // process function input data - obj.wg.Add(1) - go func(vertex pgraph.Vertex) { - node := obj.state[vertex] - defer obj.wg.Done() - defer close(node.input) - var ready bool - // the final closing output to this, closes this - for range node.notify { // new input values - // now build the struct if we can... - - ready = true // assume for now... - si := &types.Type{ - // input to functions are structs - Kind: types.KindStruct, - Map: node.handle.Info().Sig.Map, - Ord: node.handle.Info().Sig.Ord, - } - st := types.NewStruct(si) - for _, v := range incoming { - args := obj.Graph.Adjacency()[v][vertex].(*interfaces.FuncEdge).Args - from := obj.state[v] - obj.mutex.RLock() - value, exists := obj.table[v] - obj.mutex.RUnlock() - if !exists { - ready = false // nope! - break - } - - // set each arg, since one value - // could get used for multiple - // function inputs (shared edge) - for _, arg := range args { - err := st.Set(arg, value) // populate struct - if err != nil { - panic(fmt.Sprintf("struct set failure on `%s` from `%s`: %v", node, from, err)) - } - } - } - if !ready { - continue - } - - select { - case node.input <- st: // send to function - - case <-obj.closeChan: - return - } - } - }(vertex) - } - - obj.wg.Add(1) - go func(vertex pgraph.Vertex) { // run function - node := obj.state[vertex] - defer obj.wg.Done() - if obj.Debug { - obj.SafeLogf("Running func `%s`", node) - } - err := node.handle.Stream(node.ctx) - if obj.Debug { - obj.SafeLogf("Exiting func `%s`", node) - } - if err != nil { - // we closed with an error... - err := errwrap.Wrapf(err, "problem streaming func `%s`", node) - select { - case obj.ag <- err: // send to aggregate channel - - case <-obj.closeChan: - return - } - } - }(vertex) - - obj.wg.Add(1) - go func(vertex pgraph.Vertex) { // process function output data - node := obj.state[vertex] - defer obj.wg.Done() - defer obj.agDone(vertex) - outgoing := obj.Graph.OutgoingGraphVertices(vertex) // []Vertex - for value := range node.output { // read from channel - obj.mutex.RLock() - cached, exists := obj.table[vertex] - obj.mutex.RUnlock() - if !exists { // first value received - // RACE: do this AFTER value is present! - //node.loaded = true // not yet please - obj.Logf("func `%s` started", node) - } else if value.Cmp(cached) == nil { - // skip if new value is same as previous - // if this happens often, it *might* be - // a bug in the function implementation - // FIXME: do we need to disable engine - // caching when using hysteresis? - obj.Logf("func `%s` skipped", node) - continue - } - obj.mutex.Lock() - // XXX: maybe we can get rid of the table... - obj.table[vertex] = value // save the latest - node.mutex.Lock() - //if err := node.Expr.SetValue(value); err != nil { - // node.mutex.Unlock() // don't block node.String() - // panic(fmt.Sprintf("could not set value for `%s`: %+v", node, err)) - //} - node.loaded = true // set *after* value is in :) - obj.Logf("func `%s` changed", node) - node.mutex.Unlock() - obj.mutex.Unlock() - - // FIXME: will this actually prevent glitching? - // if we only notify the aggregate channel when - // we're at the bottom of the topo sort (eg: no - // outgoing vertices to notify) then we'll have - // a glitch free subtree in the programs ast... - if obj.Glitch || len(outgoing) == 0 { - select { - case obj.ag <- nil: // send to aggregate channel - - case <-obj.closeChan: - return - } - } - - // notify the receiving vertices - for _, v := range outgoing { - node := obj.state[v] - select { - case node.notify <- struct{}{}: - - case <-obj.closeChan: - return - } - } - } - // no more output values are coming... - obj.SafeLogf("func `%s` stopped", node) - - // nodes that never loaded will cause the engine to hang - if !node.loaded { - select { - case obj.ag <- fmt.Errorf("func `%s` stopped before it was loaded", node): - case <-obj.closeChan: - return - } - } - }(vertex) - } - - // send event on streamChan when any of the (aggregated) facts change - obj.wg.Add(1) - go func() { - defer obj.wg.Done() - defer close(obj.streamChan) - Loop: - for { - var err error - var ok bool - select { - case err, ok = <-obj.ag: // aggregated channel - if !ok { - break Loop // channel shutdown - } - - if !obj.loaded { - // now check if we're ready - var loaded = true // initially assume true - for _, vertex := range obj.topologicalSort { - node := obj.state[vertex] - node.mutex.RLock() - nodeLoaded := node.loaded - node.mutex.RUnlock() - if !nodeLoaded { - loaded = false // we were wrong - // TODO: add better "not loaded" reporting - if obj.Debug { - obj.Logf("not yet loaded: %s", node) - } - break - } - } - obj.loaded = loaded - - if obj.loaded { - // this causes an initial start - // signal to be sent out below, - // since the stream sender runs - if obj.Debug { - obj.Logf("loaded") - } - close(obj.loadedChan) // signal - } else { - if err == nil { - continue // not ready to send signal - } // pass errors through... - } - } - - case <-obj.closeChan: - return - } - - // send stream signal - select { - // send events or errors on streamChan, eg: func failure - case obj.streamChan <- err: // send - if err != nil { - return - } - case <-obj.closeChan: - return - } - } - }() - - return nil -} - -// agAdd registers a user on the ag channel. -func (obj *Engine) agAdd(i int) { - defer obj.agLock.Unlock() - obj.agLock.Lock() - obj.agCount += i -} - -// agDone closes the channel if we're the last one using it. -func (obj *Engine) agDone(vertex pgraph.Vertex) { - defer obj.agLock.Unlock() - obj.agLock.Lock() - node := obj.state[vertex] - node.closed = true - - // FIXME: (perf) cache this into a table which we narrow down with each - // successive call. look at the outgoing vertices that I would affect... - for _, v := range obj.Graph.OutgoingGraphVertices(vertex) { // close for each one - // now determine who provides inputs to that vertex... - var closed = true - for _, vv := range obj.Graph.IncomingGraphVertices(v) { - // are they all closed? - if !obj.state[vv].closed { - closed = false - break - } - } - if closed { // if they're all closed, we can close the input - close(obj.state[v].notify) - } - } - - if obj.agCount == 0 { - close(obj.ag) - } -} - -// Lock takes a write lock on the data that gets written to the AST, so that -// interpret/SetValue can be run without anything changing part way through. -// XXX: This API is kind of yucky, but is related to us running .String() on the -// nodes. Maybe we can avoid this somehow? -func (obj *Engine) Lock() { - obj.mutex.Lock() -} - -// Unlock takes a write lock on the data that gets written to the AST, so that -// interpret/SetValue can be run without anything changing part way through. -// XXX: This API is kind of yucky, but is related to us running .String() on the -// nodes. Maybe we can avoid this somehow? -func (obj *Engine) Unlock() { - obj.mutex.Unlock() -} - -// RLock takes a read lock on the data that gets written to the AST, so that -// interpret can be run without anything changing part way through. -func (obj *Engine) RLock() { - obj.mutex.RLock() -} - -// RUnlock frees a read lock on the data that gets written to the AST, so that -// interpret can be run without anything changing part way through. -func (obj *Engine) RUnlock() { - obj.mutex.RUnlock() -} - -// SafeLogf logs a message, although it adds a read lock around the logging in -// case a `node` argument is passed in which would set off the race detector. -func (obj *Engine) SafeLogf(format string, v ...interface{}) { - // We're adding a global mutex, because it's harder to only isolate the - // individual node specific mutexes needed since it may contain others! - if len(v) > 0 { - obj.mutex.RLock() - } - obj.Logf(format, v...) - if len(v) > 0 { - obj.mutex.RUnlock() - } -} - -// Stream returns a channel of engine events. Wait for nil events to know when -// the Table map has changed. An error event means this will shutdown shortly. -// Do not run the Table function before we've received one non-error event. -func (obj *Engine) Stream() chan error { - return obj.streamChan -} - -// Table returns a copy of the populated data table of values. We return a copy -// because since these values are constantly changing, we need an atomic -// snapshot to present to the consumer of this API. -// TODO: is this globally glitch consistent? -// TODO: do we need an API to return a single value? (wrapped in read locks) -func (obj *Engine) Table() map[pgraph.Vertex]types.Value { - obj.mutex.RLock() - defer obj.mutex.RUnlock() - table := make(map[pgraph.Vertex]types.Value) - for k, v := range obj.table { - //table[k] = v.Copy() // XXX: do we need to copy these values? - table[k] = v - } - return table -} - -// Close shuts down the function engine. It waits till everything has finished. -func (obj *Engine) Close() error { - var err error - for _, vertex := range obj.topologicalSort { // FIXME: should we do this in reverse? - node := obj.state[vertex] - node.cancel() // ctx - } - close(obj.closeChan) - obj.wg.Wait() // wait so that each func doesn't need to do this in close - obj.cancel() // free - return err -} diff --git a/lang/funcs/facts/func_test.go b/lang/funcs/facts/func_test.go deleted file mode 100644 index ced797e446..0000000000 --- a/lang/funcs/facts/func_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2023+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//go:build !root - -package facts - -import ( - "testing" - "time" - - "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/pgraph" -) - -func TestFuncGraph0(t *testing.T) { - t.Logf("Hello!") - g, _ := pgraph.NewGraph("empty") // empty graph - - obj := &funcs.Engine{ - Graph: g, - } - - t.Logf("Init...") - if err := obj.Init(); err != nil { - t.Errorf("could not init: %+v", err) - return - } - - t.Logf("Validate...") - if err := obj.Validate(); err != nil { - t.Errorf("could not validate: %+v", err) - return - } - - t.Logf("Run...") - if err := obj.Run(); err != nil { - t.Errorf("could not run: %+v", err) - return - } - - // wait for some activity - t.Logf("Stream...") - stream := obj.Stream() - t.Logf("Loop...") - br := time.After(time.Duration(5) * time.Second) -Loop: - for { - select { - case err, ok := <-stream: - if !ok { - t.Logf("Stream break...") - break Loop - } - if err != nil { - t.Logf("Error: %+v", err) - continue - } - - case <-br: - t.Logf("Break...") - t.Errorf("empty graph should have closed stream") - break Loop - } - } - - t.Logf("Closing...") - if err := obj.Close(); err != nil { - t.Errorf("could not close: %+v", err) - return - } -} diff --git a/lang/funcs/history_func.go b/lang/funcs/history_func.go index c5fbb9c83c..9614601ae8 100644 --- a/lang/funcs/history_func.go +++ b/lang/funcs/history_func.go @@ -40,6 +40,8 @@ func init() { Register(HistoryFuncName, func() interfaces.Func { return &HistoryFunc{} }) // must register the func and name } +var _ interfaces.PolyFunc = &HistoryFunc{} // ensure it meets this expectation + // HistoryFunc is special function which returns the Nth oldest value seen. It // must store up incoming values until it gets enough to return the desired one. // A restart of the program, will expunge the stored state. This obviously takes @@ -299,35 +301,35 @@ func (obj *HistoryFunc) Polymorphisms(partialType *types.Type, partialValues []t // Build takes the now known function signature and stores it so that this // function can appear to be static. That type is used to build our function // statically. -func (obj *HistoryFunc) Build(typ *types.Type) error { +func (obj *HistoryFunc) Build(typ *types.Type) (*types.Type, error) { if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 { - return fmt.Errorf("the history function needs exactly two args") + return nil, fmt.Errorf("the history function needs exactly two args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } t1, exists := typ.Map[typ.Ord[1]] if !exists || t1 == nil { - return fmt.Errorf("second arg must be specified") + return nil, fmt.Errorf("second arg must be specified") } if t1.Cmp(types.TypeInt) != nil { - return fmt.Errorf("second arg for history must be an int") + return nil, fmt.Errorf("second arg for history must be an int") } t0, exists := typ.Map[typ.Ord[0]] if !exists || t0 == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } obj.Type = t0 // type of historical value is now known! - return nil + return obj.sig(), nil } // Validate makes sure we've built our struct properly. It is usually unused for @@ -343,8 +345,7 @@ func (obj *HistoryFunc) Validate() error { func (obj *HistoryFunc) Info() *interfaces.Info { var sig *types.Type if obj.Type != nil { // don't panic if called speculatively - s := obj.Type.String() - sig = types.NewType(fmt.Sprintf("func(%s %s, %s int) %s", historyArgNameValue, s, historyArgNameIndex, s)) + sig = obj.sig() // helper } return &interfaces.Info{ Pure: false, // definitely false @@ -354,6 +355,12 @@ func (obj *HistoryFunc) Info() *interfaces.Info { } } +// helper +func (obj *HistoryFunc) sig() *types.Type { + s := obj.Type.String() + return types.NewType(fmt.Sprintf("func(%s %s, %s int) %s", historyArgNameValue, s, historyArgNameIndex, s)) +} + // Init runs some startup code for this function. func (obj *HistoryFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/listlookup_func.go b/lang/funcs/listlookup_func.go new file mode 100644 index 0000000000..706489cf6e --- /dev/null +++ b/lang/funcs/listlookup_func.go @@ -0,0 +1,489 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package funcs + +import ( + "context" + "fmt" + "math" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util/errwrap" +) + +const ( + // ListLookupFuncName is the name this function is registered as. This + // starts with an underscore so that it cannot be used from the lexer. + // XXX: change to _listlookup and add syntax in the lexer/parser + ListLookupFuncName = "listlookup" + + // arg names... + listLookupArgNameList = "list" + listLookupArgNameIndex = "index" + listLookupArgNameDefault = "default" +) + +func init() { + Register(ListLookupFuncName, func() interfaces.Func { return &ListLookupFunc{} }) // must register the func and name +} + +var _ interfaces.PolyFunc = &ListLookupFunc{} // ensure it meets this expectation + +// ListLookupFunc is a list index lookup function. If you provide a negative +// index, then it will return the default value you specified for this function. +type ListLookupFunc struct { + Type *types.Type // Kind == List, that is used as the list we lookup in + + init *interfaces.Init + last types.Value // last value received to use for diff + + result types.Value // last calculated output +} + +// String returns a simple name for this function. This is needed so this struct +// can satisfy the pgraph.Vertex interface. +func (obj *ListLookupFunc) String() string { + return ListLookupFuncName +} + +// ArgGen returns the Nth arg name for this function. +func (obj *ListLookupFunc) ArgGen(index int) (string, error) { + seq := []string{listLookupArgNameList, listLookupArgNameIndex, listLookupArgNameDefault} + if l := len(seq); index >= l { + return "", fmt.Errorf("index %d exceeds arg length of %d", index, l) + } + return seq[index], nil +} + +// Unify returns the list of invariants that this func produces. +func (obj *ListLookupFunc) Unify(expr interfaces.Expr) ([]interfaces.Invariant, error) { + var invariants []interfaces.Invariant + var invar interfaces.Invariant + + // func(list T1, index int, default T2) T2 + // (list: []T2 => T2 aka T1 => T2) + + listName, err := obj.ArgGen(0) + if err != nil { + return nil, err + } + + indexName, err := obj.ArgGen(1) + if err != nil { + return nil, err + } + + defaultName, err := obj.ArgGen(2) + if err != nil { + return nil, err + } + + dummyList := &interfaces.ExprAny{} // corresponds to the list type + dummyIndex := &interfaces.ExprAny{} // corresponds to the index type + dummyDefault := &interfaces.ExprAny{} // corresponds to the default type + dummyOut := &interfaces.ExprAny{} // corresponds to the out string + + // default type and out are the same + invar = &interfaces.EqualityInvariant{ + Expr1: dummyDefault, + Expr2: dummyOut, + } + invariants = append(invariants, invar) + + // relationship between T1 and T2 + invar = &interfaces.EqualityWrapListInvariant{ + Expr1: dummyList, + Expr2Val: dummyDefault, + } + invariants = append(invariants, invar) + + // the index has to be an int + invar = &interfaces.EqualsInvariant{ + Expr: dummyIndex, + Type: types.TypeInt, + } + invariants = append(invariants, invar) + + // full function + mapped := make(map[string]interfaces.Expr) + ordered := []string{listName, indexName, defaultName} + mapped[listName] = dummyList + mapped[indexName] = dummyIndex + mapped[defaultName] = dummyDefault + + invar = &interfaces.EqualityWrapFuncInvariant{ + Expr1: expr, // maps directly to us! + Expr2Map: mapped, + Expr2Ord: ordered, + Expr2Out: dummyOut, + } + invariants = append(invariants, invar) + + // generator function + fn := func(fnInvariants []interfaces.Invariant, solved map[interfaces.Expr]*types.Type) ([]interfaces.Invariant, error) { + for _, invariant := range fnInvariants { + // search for this special type of invariant + cfavInvar, ok := invariant.(*interfaces.CallFuncArgsValueInvariant) + if !ok { + continue + } + // did we find the mapping from us to ExprCall ? + if cfavInvar.Func != expr { + continue + } + // cfavInvar.Expr is the ExprCall! (the return pointer) + // cfavInvar.Args are the args that ExprCall uses! + if l := len(cfavInvar.Args); l != 3 { + return nil, fmt.Errorf("unable to build function with %d args", l) + } + + // add the relationship to the returned value + invar = &interfaces.EqualityInvariant{ + Expr1: cfavInvar.Expr, + Expr2: dummyOut, + } + invariants = append(invariants, invar) + + // add the relationships to the called args + invar = &interfaces.EqualityInvariant{ + Expr1: cfavInvar.Args[0], + Expr2: dummyList, + } + invariants = append(invariants, invar) + + invar = &interfaces.EqualityInvariant{ + Expr1: cfavInvar.Args[1], + Expr2: dummyIndex, + } + invariants = append(invariants, invar) + + invar = &interfaces.EqualityInvariant{ + Expr1: cfavInvar.Args[2], + Expr2: dummyDefault, + } + invariants = append(invariants, invar) + + var invariants []interfaces.Invariant + var invar interfaces.Invariant + + // If we figure out either of these types, we'll know + // the full type... + var t1 *types.Type // list type + var t2 *types.Type // list val type + + // validateArg0 checks: list T1 + validateArg0 := func(typ *types.Type) error { + if typ == nil { // unknown so far + return nil + } + + // we happen to have a list! + if k := typ.Kind; k != types.KindList { + return fmt.Errorf("unable to build function with 0th arg of kind: %s", k) + } + + if typ.Val == nil { + // programming error + return fmt.Errorf("list is missing type") + } + + if err := typ.Cmp(t1); t1 != nil && err != nil { + return errwrap.Wrapf(err, "input type was inconsistent") + } + if err := typ.Val.Cmp(t2); t2 != nil && err != nil { + return errwrap.Wrapf(err, "input val type was inconsistent") + } + + // learn! + t1 = typ + t2 = typ.Val + return nil + } + + // validateArg1 checks: list index + validateArg1 := func(typ *types.Type) error { + if typ == nil { // unknown so far + return nil + } + if typ.Kind != types.KindInt { + return errwrap.Wrapf(err, "input index type was inconsistent") + } + return nil + } + + // validateArg2 checks: list val T2 + validateArg2 := func(typ *types.Type) error { + if typ == nil { // unknown so far + return nil + } + + if err := typ.Cmp(t2); t2 != nil && err != nil { + return errwrap.Wrapf(err, "input val type was inconsistent") + } + if t1 != nil { + if err := typ.Cmp(t1.Val); err != nil { + return errwrap.Wrapf(err, "input val type was inconsistent") + } + } + t := &types.Type{ // build t1 + Kind: types.KindList, + Val: typ, // t2 + } + if t2 != nil { + if err := t.Cmp(t1); t1 != nil && err != nil { + return errwrap.Wrapf(err, "input type was inconsistent") + } + //t1 = t // learn! + } + + // learn! + t1 = t + t2 = typ + return nil + } + + if typ, err := cfavInvar.Args[0].Type(); err == nil { // is it known? + // this sets t1 and t2 on success if it learned + if err := validateArg0(typ); err != nil { + return nil, errwrap.Wrapf(err, "first list arg type is inconsistent") + } + } + if typ, exists := solved[cfavInvar.Args[0]]; exists { // alternate way to lookup type + // this sets t1 and t2 on success if it learned + if err := validateArg0(typ); err != nil { + return nil, errwrap.Wrapf(err, "first list arg type is inconsistent") + } + } + + if typ, err := cfavInvar.Args[1].Type(); err == nil { // is it known? + // this only checks if this is an int + if err := validateArg1(typ); err != nil { + return nil, errwrap.Wrapf(err, "second index arg type is inconsistent") + } + } + if typ, exists := solved[cfavInvar.Args[1]]; exists { // alternate way to lookup type + // this only checks if this is an int + if err := validateArg1(typ); err != nil { + return nil, errwrap.Wrapf(err, "second index arg type is inconsistent") + } + } + + if typ, err := cfavInvar.Args[2].Type(); err == nil { // is it known? + // this sets t1 and t2 on success if it learned + if err := validateArg2(typ); err != nil { + return nil, errwrap.Wrapf(err, "third default arg type is inconsistent") + } + } + if typ, exists := solved[cfavInvar.Args[2]]; exists { // alternate way to lookup type + // this sets t1 and t2 on success if it learned + if err := validateArg2(typ); err != nil { + return nil, errwrap.Wrapf(err, "third default arg type is inconsistent") + } + } + + // XXX: if the types aren't know statically? + + if t1 != nil { + invar := &interfaces.EqualsInvariant{ + Expr: dummyList, + Type: t1, + } + invariants = append(invariants, invar) + } + if t2 != nil { + invar := &interfaces.EqualsInvariant{ + Expr: dummyDefault, + Type: t2, + } + invariants = append(invariants, invar) + } + + // XXX: if t{1..2} are missing, we could also return a + // new generator for later if we learn new information, + // but we'd have to be careful to not do the infinitely + + // TODO: do we return this relationship with ExprCall? + invar = &interfaces.EqualityWrapCallInvariant{ + // TODO: should Expr1 and Expr2 be reversed??? + Expr1: cfavInvar.Expr, + //Expr2Func: cfavInvar.Func, // same as below + Expr2Func: expr, + } + invariants = append(invariants, invar) + + // TODO: are there any other invariants we should build? + return invariants, nil // generator return + } + // We couldn't tell the solver anything it didn't already know! + return nil, fmt.Errorf("couldn't generate new invariants") + } + invar = &interfaces.GeneratorInvariant{ + Func: fn, + } + invariants = append(invariants, invar) + + return invariants, nil +} + +// Build is run to turn the polymorphic, undetermined function, into the +// specific statically typed version. It is usually run after Unify completes, +// and must be run before Info() and any of the other Func interface methods are +// used. This function is idempotent, as long as the arg isn't changed between +// runs. +func (obj *ListLookupFunc) Build(typ *types.Type) (*types.Type, error) { + // typ is the KindFunc signature we're trying to build... + if typ.Kind != types.KindFunc { + return nil, fmt.Errorf("input type must be of kind func") + } + + if len(typ.Ord) != 3 { + return nil, fmt.Errorf("the listlookup function needs exactly three args") + } + if typ.Out == nil { + return nil, fmt.Errorf("return type of function must be specified") + } + if typ.Map == nil { + return nil, fmt.Errorf("invalid input type") + } + + tList, exists := typ.Map[typ.Ord[0]] + if !exists || tList == nil { + return nil, fmt.Errorf("first arg must be specified") + } + + tIndex, exists := typ.Map[typ.Ord[1]] + if !exists || tIndex == nil { + return nil, fmt.Errorf("second arg must be specified") + } + + tDefault, exists := typ.Map[typ.Ord[2]] + if !exists || tDefault == nil { + return nil, fmt.Errorf("third arg must be specified") + } + + if tIndex != nil && tIndex.Kind != types.KindInt { + return nil, fmt.Errorf("index must be int kind") + } + + if err := tList.Val.Cmp(tDefault); err != nil { + return nil, errwrap.Wrapf(err, "default must match list val type") + } + + if err := tList.Val.Cmp(typ.Out); err != nil { + return nil, errwrap.Wrapf(err, "return type must match list val type") + } + + obj.Type = tList // list type + return obj.sig(), nil +} + +// Validate tells us if the input struct takes a valid form. +func (obj *ListLookupFunc) Validate() error { + if obj.Type == nil { // build must be run first + return fmt.Errorf("type is still unspecified") + } + if obj.Type.Kind != types.KindList { + return fmt.Errorf("type must be a kind of list") + } + return nil +} + +// Info returns some static info about itself. Build must be called before this +// will return correct data. +func (obj *ListLookupFunc) Info() *interfaces.Info { + var sig *types.Type + if obj.Type != nil { // don't panic if called speculatively + // TODO: can obj.Type.Key or obj.Type.Val be nil (a partial) ? + sig = obj.sig() // helper + } + return &interfaces.Info{ + Pure: true, + Memo: false, + Sig: sig, // func kind + Err: obj.Validate(), + } +} + +// helper +func (obj *ListLookupFunc) sig() *types.Type { + v := obj.Type.Val.String() + return types.NewType(fmt.Sprintf("func(%s %s, %s int, %s %s) %s", listLookupArgNameList, obj.Type.String(), listLookupArgNameIndex, listLookupArgNameDefault, v, v)) +} + +// Init runs some startup code for this function. +func (obj *ListLookupFunc) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +// Stream returns the changing values that this func has over time. +func (obj *ListLookupFunc) Stream(ctx context.Context) error { + defer close(obj.init.Output) // the sender closes + for { + select { + case input, ok := <-obj.init.Input: + if !ok { + return nil // can't output any more + } + //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { + // return errwrap.Wrapf(err, "wrong function input") + //} + + if obj.last != nil && input.Cmp(obj.last) == nil { + continue // value didn't change, skip it + } + obj.last = input // store for next + + l := (input.Struct()[listLookupArgNameList]).(*types.ListValue) + index := input.Struct()[listLookupArgNameIndex].Int() + def := input.Struct()[listLookupArgNameDefault] + + // TODO: should we handle overflow by returning default? + if index > math.MaxInt { // max int size varies by arch + return fmt.Errorf("list index overflow, got: %d, max is: %d", index, math.MaxInt32) + } + + // negative index values are "not found" here! + var result types.Value + val, exists := l.Lookup(int(index)) + if exists { + result = val + } else { + result = def + } + + // if previous input was `2 + 4`, but now it + // changed to `1 + 5`, the result is still the + // same, so we can skip sending an update... + if obj.result != nil && result.Cmp(obj.result) == nil { + continue // result didn't change + } + obj.result = result // store new result + + case <-ctx.Done(): + return nil + } + + select { + case obj.init.Output <- obj.result: // send + case <-ctx.Done(): + return nil + } + } +} diff --git a/lang/funcs/maplookup_func.go b/lang/funcs/maplookup_func.go index 40c1faac25..7785d47bdc 100644 --- a/lang/funcs/maplookup_func.go +++ b/lang/funcs/maplookup_func.go @@ -42,6 +42,8 @@ func init() { Register(MapLookupFuncName, func() interfaces.Func { return &MapLookupFunc{} }) // must register the func and name } +var _ interfaces.PolyFunc = &MapLookupFunc{} // ensure it meets this expectation + // MapLookupFunc is a key map lookup function. type MapLookupFunc struct { Type *types.Type // Kind == Map, that is used as the map we lookup @@ -467,51 +469,51 @@ func (obj *MapLookupFunc) Polymorphisms(partialType *types.Type, partialValues [ // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *MapLookupFunc) Build(typ *types.Type) error { +func (obj *MapLookupFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 3 { - return fmt.Errorf("the maplookup function needs exactly three args") + return nil, fmt.Errorf("the maplookup function needs exactly three args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } tMap, exists := typ.Map[typ.Ord[0]] if !exists || tMap == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } tKey, exists := typ.Map[typ.Ord[1]] if !exists || tKey == nil { - return fmt.Errorf("second arg must be specified") + return nil, fmt.Errorf("second arg must be specified") } tDef, exists := typ.Map[typ.Ord[2]] if !exists || tDef == nil { - return fmt.Errorf("third arg must be specified") + return nil, fmt.Errorf("third arg must be specified") } if err := tMap.Key.Cmp(tKey); err != nil { - return errwrap.Wrapf(err, "key must match map key type") + return nil, errwrap.Wrapf(err, "key must match map key type") } if err := tMap.Val.Cmp(tDef); err != nil { - return errwrap.Wrapf(err, "default must match map val type") + return nil, errwrap.Wrapf(err, "default must match map val type") } if err := tMap.Val.Cmp(typ.Out); err != nil { - return errwrap.Wrapf(err, "return type must match map val type") + return nil, errwrap.Wrapf(err, "return type must match map val type") } obj.Type = tMap // map type - return nil + return obj.sig(), nil } // Validate tells us if the input struct takes a valid form. @@ -528,21 +530,26 @@ func (obj *MapLookupFunc) Validate() error { // Info returns some static info about itself. Build must be called before this // will return correct data. func (obj *MapLookupFunc) Info() *interfaces.Info { - var typ *types.Type + var sig *types.Type if obj.Type != nil { // don't panic if called speculatively // TODO: can obj.Type.Key or obj.Type.Val be nil (a partial) ? - k := obj.Type.Key.String() - v := obj.Type.Val.String() - typ = types.NewType(fmt.Sprintf("func(%s %s, %s %s, %s %s) %s", mapLookupArgNameMap, obj.Type.String(), mapLookupArgNameKey, k, mapLookupArgNameDef, v, v)) + sig = obj.sig() // helper } return &interfaces.Info{ Pure: true, Memo: false, - Sig: typ, // func kind + Sig: sig, // func kind Err: obj.Validate(), } } +// helper +func (obj *MapLookupFunc) sig() *types.Type { + k := obj.Type.Key.String() + v := obj.Type.Val.String() + return types.NewType(fmt.Sprintf("func(%s %s, %s %s, %s %s) %s", mapLookupArgNameMap, obj.Type.String(), mapLookupArgNameKey, k, mapLookupArgNameDef, v, v)) +} + // Init runs some startup code for this function. func (obj *MapLookupFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/operator_func.go b/lang/funcs/operator_func.go index 99b34482bc..018c54c645 100644 --- a/lang/funcs/operator_func.go +++ b/lang/funcs/operator_func.go @@ -336,6 +336,8 @@ func init() { Register(OperatorFuncName, func() interfaces.Func { return &OperatorFunc{} }) // must register the func and name } +var _ interfaces.PolyFunc = &OperatorFunc{} // ensure it meets this expectation + // OperatorFuncs maps an operator to a list of callable function values. var OperatorFuncs = make(map[string][]*types.FuncValue) // must initialize @@ -791,17 +793,46 @@ func (obj *OperatorFunc) Polymorphisms(partialType *types.Type, partialValues [] // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *OperatorFunc) Build(typ *types.Type) error { +func (obj *OperatorFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if len(typ.Ord) < 1 { - return fmt.Errorf("the operator function needs at least 1 arg") + return nil, fmt.Errorf("the operator function needs at least 1 arg") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") + } + if typ.Kind != types.KindFunc { + return nil, fmt.Errorf("unexpected build kind of: %v", typ.Kind) } - obj.Type = typ // func type - return nil + // Change arg names to be what we expect... + if _, exists := typ.Map[typ.Ord[0]]; !exists { + return nil, fmt.Errorf("invalid build type") + } + + //newTyp := typ.Copy() + newTyp := &types.Type{ + Kind: typ.Kind, // copy + Map: make(map[string]*types.Type), // new + Ord: []string{}, // new + Out: typ.Out, // copy + } + for i, x := range typ.Ord { // remap arg names + //argName := util.NumToAlpha(i - 1) + //if i == 0 { + // argName = operatorArgName + //} + argName, err := obj.ArgGen(i) + if err != nil { + return nil, err + } + + newTyp.Map[argName] = typ.Map[x] + newTyp.Ord = append(newTyp.Ord, argName) + } + + obj.Type = newTyp // func type + return obj.Type, nil } // Validate tells us if the input struct takes a valid form. @@ -852,10 +883,27 @@ func (obj *OperatorFunc) Stream(ctx context.Context) error { } obj.last = input // store for next + // programming error safety check... + programmingError := false + keys := []string{} + for k := range input.Struct() { + keys = append(keys, k) + if !util.StrInList(k, obj.Type.Ord) { + programmingError = true + } + } + if programmingError { + return fmt.Errorf("bad args, got: %v, want: %v", keys, obj.Type.Ord) + } + // build up arg list args := []types.Value{} for _, name := range obj.Type.Ord { - v := input.Struct()[name] + v, exists := input.Struct()[name] + if !exists { + // programming error + return fmt.Errorf("function engine was early, missing arg: %s", name) + } if name == operatorArgName { op = v.Str() continue // skip over the operator arg @@ -864,7 +912,8 @@ func (obj *OperatorFunc) Stream(ctx context.Context) error { } if op == "" { - return fmt.Errorf("operator cannot be empty") + // programming error + return fmt.Errorf("operator cannot be empty, args: %v", keys) } // operator selection is dynamic now, although mostly it // should not change... to do so is probably uncommon... diff --git a/lang/funcs/simple/simple.go b/lang/funcs/simple/simple.go index f9d4be05b1..28e83e711d 100644 --- a/lang/funcs/simple/simple.go +++ b/lang/funcs/simple/simple.go @@ -84,7 +84,7 @@ type WrappedFunc struct { // String returns a simple name for this function. This is needed so this struct // can satisfy the pgraph.Vertex interface. func (obj *WrappedFunc) String() string { - return fmt.Sprintf("%s@%p", obj.Name, obj) // be more unique! + return fmt.Sprintf("%s @ %p", obj.Name, obj) // be more unique! } // ArgGen returns the Nth arg name for this function. diff --git a/lang/funcs/simplepoly/simplepoly.go b/lang/funcs/simplepoly/simplepoly.go index 2f9fba7952..56d998dc82 100644 --- a/lang/funcs/simplepoly/simplepoly.go +++ b/lang/funcs/simplepoly/simplepoly.go @@ -128,6 +128,8 @@ func consistentArgs(fns []*types.FuncValue) ([]string, error) { return seq, nil } +var _ interfaces.PolyFunc = &WrappedFunc{} // ensure it meets this expectation + // WrappedFunc is a scaffolding function struct which fulfills the boiler-plate // for the function API, but that can run a very simple, static, pure, // polymorphic function. @@ -478,23 +480,31 @@ func (obj *WrappedFunc) Polymorphisms(partialType *types.Type, partialValues []t // specific statically typed version. It is usually run after Unify completes, // and must be run before Info() and any of the other Func interface methods are // used. -func (obj *WrappedFunc) Build(typ *types.Type) error { +func (obj *WrappedFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... index, err := langutil.FnMatch(typ, obj.Fns) if err != nil { - return err + return nil, err } - obj.buildFunction(typ, index) // found match at this index + newTyp := obj.buildFunction(typ, index) // found match at this index - return nil + return newTyp, nil } // buildFunction builds our concrete static function, from the potentially // abstract, possibly variant containing list of functions. -func (obj *WrappedFunc) buildFunction(typ *types.Type, ix int) { - obj.fn = obj.Fns[ix].Copy().(*types.FuncValue) +func (obj *WrappedFunc) buildFunction(typ *types.Type, ix int) *types.Type { + cp := obj.Fns[ix].Copy() + fn, ok := cp.(*types.FuncValue) + if !ok { + panic("unexpected type") + } + obj.fn = fn + // FIXME: if obj.fn.T == nil {} // occasionally this is nil, is it a bug? obj.fn.T = typ.Copy() // overwrites any contained "variant" type + + return obj.fn.T } // Validate makes sure we've built our struct properly. It is usually unused for diff --git a/lang/funcs/structlookup_func.go b/lang/funcs/structlookup_func.go index 2b3ae3e680..283ed4cb2c 100644 --- a/lang/funcs/structlookup_func.go +++ b/lang/funcs/structlookup_func.go @@ -41,6 +41,8 @@ func init() { Register(StructLookupFuncName, func() interfaces.Func { return &StructLookupFunc{} }) // must register the func and name } +var _ interfaces.PolyFunc = &StructLookupFunc{} // ensure it meets this expectation + // StructLookupFunc is a struct field lookup function. type StructLookupFunc struct { Type *types.Type // Kind == Struct, that is used as the struct we lookup @@ -394,33 +396,33 @@ func (obj *StructLookupFunc) Polymorphisms(partialType *types.Type, partialValue // and must be run before Info() and any of the other Func interface methods are // used. This function is idempotent, as long as the arg isn't changed between // runs. -func (obj *StructLookupFunc) Build(typ *types.Type) error { +func (obj *StructLookupFunc) Build(typ *types.Type) (*types.Type, error) { // typ is the KindFunc signature we're trying to build... if typ.Kind != types.KindFunc { - return fmt.Errorf("input type must be of kind func") + return nil, fmt.Errorf("input type must be of kind func") } if len(typ.Ord) != 2 { - return fmt.Errorf("the structlookup function needs exactly two args") + return nil, fmt.Errorf("the structlookup function needs exactly two args") } if typ.Out == nil { - return fmt.Errorf("return type of function must be specified") + return nil, fmt.Errorf("return type of function must be specified") } if typ.Map == nil { - return fmt.Errorf("invalid input type") + return nil, fmt.Errorf("invalid input type") } tStruct, exists := typ.Map[typ.Ord[0]] if !exists || tStruct == nil { - return fmt.Errorf("first arg must be specified") + return nil, fmt.Errorf("first arg must be specified") } tField, exists := typ.Map[typ.Ord[1]] if !exists || tField == nil { - return fmt.Errorf("second arg must be specified") + return nil, fmt.Errorf("second arg must be specified") } if err := tField.Cmp(types.TypeStr); err != nil { - return errwrap.Wrapf(err, "field must be an str") + return nil, errwrap.Wrapf(err, "field must be an str") } // NOTE: We actually don't know which field this is, only its type! we @@ -429,7 +431,8 @@ func (obj *StructLookupFunc) Build(typ *types.Type) error { // struct. obj.Type = tStruct // struct type obj.Out = typ.Out // type of return value - return nil + + return obj.sig(), nil } // Validate tells us if the input struct takes a valid form. @@ -458,7 +461,7 @@ func (obj *StructLookupFunc) Info() *interfaces.Info { var sig *types.Type if obj.Type != nil { // don't panic if called speculatively // TODO: can obj.Out be nil (a partial) ? - sig = types.NewType(fmt.Sprintf("func(%s %s, %s str) %s", structLookupArgNameStruct, obj.Type.String(), structLookupArgNameField, obj.Out.String())) + sig = obj.sig() // helper } return &interfaces.Info{ Pure: true, @@ -468,6 +471,11 @@ func (obj *StructLookupFunc) Info() *interfaces.Info { } } +// helper +func (obj *StructLookupFunc) sig() *types.Type { + return types.NewType(fmt.Sprintf("func(%s %s, %s str) %s", structLookupArgNameStruct, obj.Type.String(), structLookupArgNameField, obj.Out.String())) +} + // Init runs some startup code for this function. func (obj *StructLookupFunc) Init(init *interfaces.Init) error { obj.init = init diff --git a/lang/funcs/structs/call.go b/lang/funcs/structs/call.go index 80285000c3..bcf2cb8825 100644 --- a/lang/funcs/structs/call.go +++ b/lang/funcs/structs/call.go @@ -23,30 +23,38 @@ import ( "github.com/purpleidea/mgmt/lang/interfaces" "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/lang/types/full" "github.com/purpleidea/mgmt/util/errwrap" ) const ( // CallFuncName is the unique name identifier for this function. CallFuncName = "call" + + // CallFuncArgNameFunction is the name for the edge which connects the + // input function to CallFunc. + CallFuncArgNameFunction = "fn" ) -// CallFunc is a function that takes in a function and all the args, and passes -// through the results of running the function call. +// CallFunc receives a function from upstream, but not the arguments. Instead, +// the Funcs which emit those arguments must be specified at construction time. +// The arguments are connected to the received FuncValues in such a way that +// CallFunc emits the result of applying the function to the arguments. type CallFunc struct { - Type *types.Type // this is the type of the var's value that we hold - FuncType *types.Type - Edge string // name of the edge used (typically starts with: `call:`) - //Func interfaces.Func // this isn't actually used in the Stream :/ - //Fn *types.FuncValue // pass in the actual function instead of Edge - - // Indexed specifies that args are accessed by index instead of name. - // This is currently unused. - Indexed bool - - init *interfaces.Init - last types.Value // last value received to use for diff - result types.Value // last calculated output + Type *types.Type // the type of the result of applying the function + FuncType *types.Type // the type of the function + EdgeName string + + ArgVertices []interfaces.Func + + init *interfaces.Init + + lastFuncValue *full.FuncValue // remember the last function value + + // outputChan is an initially-nil channel from which we receive output + // lists from the subgraph. This channel is reset when the subgraph is + // recreated. + outputChan chan types.Value } // String returns a simple name for this function. This is needed so this struct @@ -63,15 +71,14 @@ func (obj *CallFunc) Validate() error { if obj.FuncType == nil { return fmt.Errorf("must specify a func type") } - // TODO: maybe we can remove this if we use this for core functions... - if obj.Edge == "" { - return fmt.Errorf("must specify an edge name") - } typ := obj.FuncType // we only care about the output type of calling our func if err := obj.Type.Cmp(typ.Out); err != nil { return errwrap.Wrapf(err, "call expr type must match func out type") } + if len(obj.ArgVertices) != len(typ.Ord) { + return fmt.Errorf("number of arg Funcs must match number of func args in the type") + } return nil } @@ -79,25 +86,8 @@ func (obj *CallFunc) Validate() error { // Info returns some static info about itself. func (obj *CallFunc) Info() *interfaces.Info { var typ *types.Type - if obj.Type != nil { // don't panic if called speculatively - typ = &types.Type{ - Kind: types.KindFunc, // function type - Map: make(map[string]*types.Type), - Ord: []string{}, - Out: obj.Type, // this is the output type for the expression - } - - sig := obj.FuncType - if obj.Edge != "" { - typ.Map[obj.Edge] = sig // we get a function in - typ.Ord = append(typ.Ord, obj.Edge) - } - - // add any incoming args - for _, key := range sig.Ord { // sig.Out, not sig! - typ.Map[key] = sig.Map[key] - typ.Ord = append(typ.Ord, key) - } + if obj.Type != nil && obj.FuncType != nil { // don't panic if called speculatively + typ = types.NewType(fmt.Sprintf("func(%s %s) %s", obj.EdgeName, obj.FuncType, obj.Type)) } return &interfaces.Info{ @@ -111,6 +101,7 @@ func (obj *CallFunc) Info() *interfaces.Info { // Init runs some startup code for this composite function. func (obj *CallFunc) Init(init *interfaces.Init) error { obj.init = init + obj.lastFuncValue = nil return nil } @@ -119,65 +110,119 @@ func (obj *CallFunc) Init(init *interfaces.Init) error { // on the changing inputs to that value. func (obj *CallFunc) Stream(ctx context.Context) error { defer close(obj.init.Output) // the sender closes + + obj.outputChan = nil + + defer func() { + obj.init.Txn.Reverse() + }() + + canReceiveMoreFuncValues := true + canReceiveMoreOutputValues := true for { + + if !canReceiveMoreFuncValues && !canReceiveMoreOutputValues { + // break + return nil + } + select { case input, ok := <-obj.init.Input: if !ok { - return nil // can't output any more + obj.init.Input = nil // block looping back here + canReceiveMoreFuncValues = false + continue } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it + + value, exists := input.Struct()[obj.EdgeName] + if !exists { + return fmt.Errorf("programming error, can't find edge") } - obj.last = input // store for next - st := input.(*types.StructValue) // must be! + newFuncValue, ok := value.(*full.FuncValue) + if !ok { + return fmt.Errorf("programming error, can't convert to *FuncValue") + } - // get the function - fn, exists := st.Lookup(obj.Edge) - if !exists { - return fmt.Errorf("missing expected input argument `%s`", obj.Edge) + // It's important to have this compare step to avoid + // redundant graph replacements which slow things down, + // but also cause the engine to lock, which can preempt + // the process scheduler, which can cause duplicate or + // unnecessary re-sending of values here, which causes + // the whole process to repeat ad-nauseum. + if newFuncValue == obj.lastFuncValue { + continue } + // If we have a new function, then we need to replace + // the subgraph with a new one that uses the new + // function. + obj.lastFuncValue = newFuncValue - // get the arguments to call the function - args := []types.Value{} - typ := obj.FuncType - for ix, key := range typ.Ord { // sig! - if obj.Indexed { - key = fmt.Sprintf("%d", ix) - } - value, exists := st.Lookup(key) - // TODO: replace with: - //value, exists := st.Lookup(fmt.Sprintf("arg:%s", key)) - if !exists { - return fmt.Errorf("missing expected input argument `%s`", key) - } - args = append(args, value) + if err := obj.replaceSubGraph(newFuncValue); err != nil { + return errwrap.Wrapf(err, "could not replace subgraph") } + canReceiveMoreOutputValues = true + continue - // actually call it - result, err := fn.(*types.FuncValue).Call(args) - if err != nil { - return errwrap.Wrapf(err, "error calling function") + case outputValue, ok := <-obj.outputChan: + // send the new output value downstream + if !ok { + obj.outputChan = nil + canReceiveMoreOutputValues = false + continue } - // skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change + // send to the output + select { + case obj.init.Output <- outputValue: + case <-ctx.Done(): + return nil } - obj.result = result // store new result case <-ctx.Done(): return nil } + } +} - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } +func (obj *CallFunc) replaceSubGraph(newFuncValue *full.FuncValue) error { + // Create a subgraph which looks as follows. Most of the nodes are + // elided because we don't know which nodes the FuncValues will create. + // + // digraph { + // ArgVertices[0] -> ... + // ArgVertices[1] -> ... + // ArgVertices[2] -> ... + // + // outputFunc -> "subgraphOutput" + // } + + // delete the old subgraph + if err := obj.init.Txn.Reverse(); err != nil { + return errwrap.Wrapf(err, "could not Reverse") + } + + // create the new subgraph + // This passed in Txn has AddVertex, AddEdge, and possibly AddGraph + // methods called on it. Nothing else. It will _not_ call Commit or + // Reverse. It adds to the graph, and our Commit and Reverse operations + // are the ones that actually make the change. + outputFunc, err := newFuncValue.Call(obj.init.Txn, obj.ArgVertices) + if err != nil { + return errwrap.Wrapf(err, "could not call newFuncValue.Call()") + } + + obj.outputChan = make(chan types.Value) + edgeName := ChannelBasedSinkFuncArgName + subgraphOutput := &ChannelBasedSinkFunc{ + Name: "subgraphOutput", + Target: obj, + EdgeName: edgeName, + Chan: obj.outputChan, + Type: obj.Type, } + edge := &interfaces.FuncEdge{Args: []string{edgeName}} + obj.init.Txn.AddVertex(subgraphOutput) + obj.init.Txn.AddEdge(outputFunc, subgraphOutput, edge) + return obj.init.Txn.Commit() } diff --git a/lang/funcs/structs/channel_based_sink.go b/lang/funcs/structs/channel_based_sink.go new file mode 100644 index 0000000000..4013c2e009 --- /dev/null +++ b/lang/funcs/structs/channel_based_sink.go @@ -0,0 +1,129 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package structs + +import ( + "context" + "fmt" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" +) + +const ( + // ChannelBasedSinkFuncArgName is the name for the edge which connects + // the input value to ChannelBasedSinkFunc. + ChannelBasedSinkFuncArgName = "channelBasedSinkFuncArg" +) + +// ChannelBasedSinkFunc is a Func which receives values from upstream nodes and +// emits them to a golang channel. +type ChannelBasedSinkFunc struct { + Name string + EdgeName string + Target interfaces.Func // for drawing dashed edges in the Graphviz visualization + + Chan chan types.Value + Type *types.Type + + init *interfaces.Init + last types.Value // last value received to use for diff +} + +// String returns a simple name for this function. This is needed so this struct +// can satisfy the pgraph.Vertex interface. +func (obj *ChannelBasedSinkFunc) String() string { + return obj.Name +} + +// ArgGen returns the Nth arg name for this function. +func (obj *ChannelBasedSinkFunc) ArgGen(index int) (string, error) { + if index != 0 { + return "", fmt.Errorf("the ChannelBasedSinkFunc only has one argument") + } + return obj.EdgeName, nil +} + +// Validate makes sure we've built our struct properly. It is usually unused for +// normal functions that users can use directly. +func (obj *ChannelBasedSinkFunc) Validate() error { + if obj.Chan == nil { + return fmt.Errorf("the Chan was not set") + } + return nil +} + +// Info returns some static info about itself. +func (obj *ChannelBasedSinkFunc) Info() *interfaces.Info { + return &interfaces.Info{ + Pure: false, + Memo: false, + Sig: types.NewType(fmt.Sprintf("func(%s %s) %s", obj.EdgeName, obj.Type, obj.Type)), + Err: obj.Validate(), + } +} + +// Init runs some startup code for this function. +func (obj *ChannelBasedSinkFunc) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +// Stream returns the changing values that this func has over time. +func (obj *ChannelBasedSinkFunc) Stream(ctx context.Context) error { + defer close(obj.init.Output) // the sender closes + defer close(obj.Chan) // the sender closes + + for { + select { + case input, ok := <-obj.init.Input: + if !ok { + return nil // can't output any more + } + + value, exists := input.Struct()[obj.EdgeName] + if !exists { + return fmt.Errorf("programming error, can't find edge") + } + + if obj.last != nil && value.Cmp(obj.last) == nil { + continue // value didn't change, skip it + } + obj.last = value // store so we can send after this select + + case <-ctx.Done(): + return nil + } + + select { + case obj.Chan <- obj.last: // send + case <-ctx.Done(): + return nil + } + + // Also send the value downstream. If we don't, then when we + // close the Output channel, the function engine is going to + // complain that we closed that channel without sending it any + // value. + select { + case obj.init.Output <- obj.last: // send + case <-ctx.Done(): + return nil + } + } +} diff --git a/lang/funcs/structs/channel_based_source.go b/lang/funcs/structs/channel_based_source.go new file mode 100644 index 0000000000..411968d290 --- /dev/null +++ b/lang/funcs/structs/channel_based_source.go @@ -0,0 +1,103 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package structs + +import ( + "context" + "fmt" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" +) + +// ChannelBasedSourceFunc is a Func which receives values from a golang channel +// and emits them to the downstream nodes. +type ChannelBasedSourceFunc struct { + Name string + Source interfaces.Func // for drawing dashed edges in the Graphviz visualization + + Chan chan types.Value + Type *types.Type + + init *interfaces.Init + last types.Value // last value received to use for diff +} + +// String returns a simple name for this function. This is needed so this struct +// can satisfy the pgraph.Vertex interface. +func (obj *ChannelBasedSourceFunc) String() string { + return "ChannelBasedSourceFunc" +} + +// ArgGen returns the Nth arg name for this function. +func (obj *ChannelBasedSourceFunc) ArgGen(index int) (string, error) { + return "", fmt.Errorf("the ChannelBasedSourceFunc doesn't have any arguments") +} + +// Validate makes sure we've built our struct properly. It is usually unused for +// normal functions that users can use directly. +func (obj *ChannelBasedSourceFunc) Validate() error { + if obj.Chan == nil { + return fmt.Errorf("the Chan was not set") + } + return nil +} + +// Info returns some static info about itself. +func (obj *ChannelBasedSourceFunc) Info() *interfaces.Info { + return &interfaces.Info{ + Pure: false, + Memo: false, + Sig: types.NewType(fmt.Sprintf("func() %s", obj.Type)), + Err: obj.Validate(), + } +} + +// Init runs some startup code for this function. +func (obj *ChannelBasedSourceFunc) Init(init *interfaces.Init) error { + obj.init = init + return nil +} + +// Stream returns the changing values that this func has over time. +func (obj *ChannelBasedSourceFunc) Stream(ctx context.Context) error { + defer close(obj.init.Output) // the sender closes + + for { + select { + case input, ok := <-obj.Chan: + if !ok { + return nil // can't output any more + } + + //if obj.last != nil && input.Cmp(obj.last) == nil { + // continue // value didn't change, skip it + //} + obj.last = input // store so we can send after this select + + case <-ctx.Done(): + return nil + } + + select { + case obj.init.Output <- obj.last: // send + case <-ctx.Done(): + return nil + } + } +} diff --git a/lang/funcs/structs/const.go b/lang/funcs/structs/const.go index 8f18c6ab24..4a718109ea 100644 --- a/lang/funcs/structs/const.go +++ b/lang/funcs/structs/const.go @@ -32,7 +32,8 @@ const ( // ConstFunc is a function that returns the constant value passed to Value. type ConstFunc struct { - Value types.Value + Value types.Value + NameHint string init *interfaces.Init } @@ -40,7 +41,12 @@ type ConstFunc struct { // String returns a simple name for this function. This is needed so this struct // can satisfy the pgraph.Vertex interface. func (obj *ConstFunc) String() string { + if obj.NameHint != "" { + return obj.NameHint + } return ConstFuncName + //return fmt.Sprintf("%s: %s(%s)", ConstFuncName, obj.Value.Type().String(), obj.Value.String()) + //return fmt.Sprintf("%s(%s)", obj.Value.Type().String(), obj.Value.String()) } // Validate makes sure we've built our struct properly. diff --git a/lang/funcs/structs/function.go b/lang/funcs/structs/function.go deleted file mode 100644 index 330fa29668..0000000000 --- a/lang/funcs/structs/function.go +++ /dev/null @@ -1,206 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2023+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package structs - -import ( - "context" - "fmt" - - "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - "github.com/purpleidea/mgmt/util/errwrap" -) - -const ( - // FunctionFuncName is the unique name identifier for this function. - FunctionFuncName = "function" -) - -// FunctionFunc is a function that passes through the function body it receives. -type FunctionFunc struct { - Type *types.Type // this is the type of the function that we hold - Edge string // name of the edge used (typically "body") - Func interfaces.Func - Fn *types.FuncValue - - init *interfaces.Init - last types.Value // last value received to use for diff - result types.Value // last calculated output -} - -// String returns a simple name for this function. This is needed so this struct -// can satisfy the pgraph.Vertex interface. -func (obj *FunctionFunc) String() string { - return FunctionFuncName -} - -// fn returns the function that wraps the Func interface if that API is used. -func (obj *FunctionFunc) fn() (*types.FuncValue, error) { - fn := func(args []types.Value) (types.Value, error) { - // FIXME: can we run a recursive engine - // to support running non-pure functions? - if !obj.Func.Info().Pure { - return nil, fmt.Errorf("only pure functions can be used by value") - } - - // XXX: this might not be needed anymore... - return funcs.PureFuncExec(obj.Func, args) - } - - result := types.NewFunc(obj.Type) // new func - if err := result.Set(fn); err != nil { - return nil, errwrap.Wrapf(err, "can't build func from built-in") - } - - return result, nil -} - -// Validate makes sure we've built our struct properly. -func (obj *FunctionFunc) Validate() error { - if obj.Type == nil { - return fmt.Errorf("must specify a type") - } - if obj.Type.Kind != types.KindFunc { - return fmt.Errorf("can't use type `%s`", obj.Type.String()) - } - if obj.Edge == "" && obj.Func == nil && obj.Fn == nil { - return fmt.Errorf("must specify an Edge, Func, or Fn") - } - - if obj.Fn != nil && obj.Fn.Type() != obj.Type { - return fmt.Errorf("type of Fn did not match input Type") - } - - return nil -} - -// Info returns some static info about itself. -func (obj *FunctionFunc) Info() *interfaces.Info { - var typ *types.Type - if obj.Type != nil { // don't panic if called speculatively - typ = &types.Type{ - Kind: types.KindFunc, // function type - Map: make(map[string]*types.Type), - Ord: []string{}, - Out: obj.Type, // after the function is called it's this... - } - - // type of body is what we'd get by running the function (what's inside) - if obj.Edge != "" { - typ.Map[obj.Edge] = obj.Type.Out - typ.Ord = append(typ.Ord, obj.Edge) - } - } - - pure := true // assume true - if obj.Func != nil { - pure = obj.Func.Info().Pure - } - - return &interfaces.Info{ - Pure: pure, // TODO: can we guarantee this? - Memo: false, // TODO: ??? - Sig: typ, - Err: obj.Validate(), - } -} - -// Init runs some startup code for this composite function. -func (obj *FunctionFunc) Init(init *interfaces.Init) error { - obj.init = init - return nil -} - -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *FunctionFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - if obj.Edge != "" { // then it's not a built-in - return nil // can't output any more - } - - var result *types.FuncValue - - if obj.Fn != nil { - result = obj.Fn - } else { - var err error - result, err = obj.fn() - if err != nil { - return err - } - } - - // if we never had input args, send the function - select { - case obj.init.Output <- result: // send - // pass - case <-ctx.Done(): - return nil - } - - return nil - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - var result types.Value - - st := input.(*types.StructValue) // must be! - value, exists := st.Lookup(obj.Edge) // single argName - if !exists { - return fmt.Errorf("missing expected input argument `%s`", obj.Edge) - } - - result = obj.Type.New() // new func - fn := func([]types.Value) (types.Value, error) { - return value, nil - } - if err := result.(*types.FuncValue).Set(fn); err != nil { - return errwrap.Wrapf(err, "can't build func with body") - } - - // skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} diff --git a/lang/funcs/structs/util.go b/lang/funcs/structs/util.go new file mode 100644 index 0000000000..d5b98c389a --- /dev/null +++ b/lang/funcs/structs/util.go @@ -0,0 +1,74 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package structs + +import ( + "github.com/purpleidea/mgmt/lang/funcs/simple" + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/lang/types/full" +) + +// In the following set of conversion functions, a "constant" Func is a node +// with in-degree zero which always outputs the same function value, while a +// "direct" Func is a node with one upstream node for each of the function's +// arguments. + +// FuncValueToConstFunc transforms a *full.FuncValue into an interfaces.Func +// which is implemented by &ConstFunc{}. +func FuncValueToConstFunc(fv *full.FuncValue) interfaces.Func { + return &ConstFunc{ + Value: fv, + NameHint: "FuncValue", + } +} + +// SimpleFnToDirectFunc transforms a name and *types.FuncValue into an +// interfaces.Func which is implemented by &simple.WrappedFunc{}. +func SimpleFnToDirectFunc(name string, fv *types.FuncValue) interfaces.Func { + return &simple.WrappedFunc{ + Name: name, + Fn: fv, + } +} + +// SimpleFnToFuncValue transforms a name and *types.FuncValue into a +// *full.FuncValue. +func SimpleFnToFuncValue(name string, fv *types.FuncValue) *full.FuncValue { + return &full.FuncValue{ + V: func(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + wrappedFunc := SimpleFnToDirectFunc(name, fv) + txn.AddVertex(wrappedFunc) + for i, arg := range args { + argName := fv.T.Ord[i] + txn.AddEdge(arg, wrappedFunc, &interfaces.FuncEdge{ + Args: []string{argName}, + }) + } + return wrappedFunc, nil + }, + T: fv.T, + } +} + +// SimpleFnToConstFunc transforms a name and *types.FuncValue into an +// interfaces.Func which is implemented by FuncValueToConstFunc and +// SimpleFnToFuncValue. +func SimpleFnToConstFunc(name string, fv *types.FuncValue) interfaces.Func { + return FuncValueToConstFunc(SimpleFnToFuncValue(name, fv)) +} diff --git a/lang/funcs/structs/var.go b/lang/funcs/structs/var.go deleted file mode 100644 index 8fef13306d..0000000000 --- a/lang/funcs/structs/var.go +++ /dev/null @@ -1,133 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2023+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -package structs - -import ( - "context" - "fmt" - - "github.com/purpleidea/mgmt/lang/interfaces" - "github.com/purpleidea/mgmt/lang/types" - //"github.com/purpleidea/mgmt/util/errwrap" -) - -const ( - // VarFuncName is the unique name identifier for this function. - VarFuncName = "var" -) - -// VarFunc is a function that passes through a function that came from a bind -// lookup. It exists so that the reactive function engine type checks correctly. -type VarFunc struct { - Type *types.Type // this is the type of the var's value that we hold - Edge string // name of the edge used - //Func interfaces.Func // this isn't actually used in the Stream :/ - - init *interfaces.Init - last types.Value // last value received to use for diff - result types.Value // last calculated output -} - -// String returns a simple name for this function. This is needed so this struct -// can satisfy the pgraph.Vertex interface. -func (obj *VarFunc) String() string { - return VarFuncName -} - -// Validate makes sure we've built our struct properly. -func (obj *VarFunc) Validate() error { - if obj.Type == nil { - return fmt.Errorf("must specify a type") - } - if obj.Edge == "" { - return fmt.Errorf("must specify an edge name") - } - return nil -} - -// Info returns some static info about itself. -func (obj *VarFunc) Info() *interfaces.Info { - var typ *types.Type - if obj.Type != nil { // don't panic if called speculatively - typ = &types.Type{ - Kind: types.KindFunc, // function type - Map: map[string]*types.Type{obj.Edge: obj.Type}, - Ord: []string{obj.Edge}, - Out: obj.Type, // this is the output type for the expression - } - } - - return &interfaces.Info{ - Pure: true, - Memo: false, // TODO: ??? - Sig: typ, - Err: obj.Validate(), - } -} - -// Init runs some startup code for this composite function. -func (obj *VarFunc) Init(init *interfaces.Init) error { - obj.init = init - return nil -} - -// Stream takes an input struct in the format as described in the Func and Graph -// methods of the Expr, and returns the actual expected value as a stream based -// on the changing inputs to that value. -func (obj *VarFunc) Stream(ctx context.Context) error { - defer close(obj.init.Output) // the sender closes - for { - select { - case input, ok := <-obj.init.Input: - if !ok { - return nil // can't output any more - } - //if err := input.Type().Cmp(obj.Info().Sig.Input); err != nil { - // return errwrap.Wrapf(err, "wrong function input") - //} - if obj.last != nil && input.Cmp(obj.last) == nil { - continue // value didn't change, skip it - } - obj.last = input // store for next - - var result types.Value - st := input.(*types.StructValue) // must be! - value, exists := st.Lookup(obj.Edge) - if !exists { - return fmt.Errorf("missing expected input argument `%s`", obj.Edge) - } - result = value - - // skip sending an update... - if obj.result != nil && result.Cmp(obj.result) == nil { - continue // result didn't change - } - obj.result = result // store new result - - case <-ctx.Done(): - return nil - } - - select { - case obj.init.Output <- obj.result: // send - // pass - case <-ctx.Done(): - return nil - } - } -} diff --git a/lang/gapi/gapi.go b/lang/gapi/gapi.go index 1f840c2c3f..1b975158d9 100644 --- a/lang/gapi/gapi.go +++ b/lang/gapi/gapi.go @@ -20,6 +20,7 @@ package gapi import ( "bytes" + "context" "fmt" "strings" "sync" @@ -61,7 +62,11 @@ func init() { type GAPI struct { InputURI string // input URI of code file system to run - lang *lang.Lang // lang struct + lang *lang.Lang // lang struct + wgRun *sync.WaitGroup + ctx context.Context + cancel func() + reterr error // this data struct is only available *after* Init, so as a result, it // can not be used inside the Cli(...) method. @@ -472,13 +477,27 @@ func (obj *GAPI) LangInit() error { if err := obj.lang.Init(); err != nil { return errwrap.Wrapf(err, "can't init the lang") } + + // XXX: I'm certain I've probably got a deadlock or race somewhere here + // or in lib/main.go so we'll fix it with an API fixup and rewrite soon + obj.wgRun = &sync.WaitGroup{} + obj.ctx, obj.cancel = context.WithCancel(context.Background()) + obj.wgRun.Add(1) + go func() { + defer obj.wgRun.Done() + obj.reterr = obj.lang.Run(obj.ctx) + }() + return nil } // LangClose is a wrapper around the lang Close method. func (obj *GAPI) LangClose() error { if obj.lang != nil { - err := obj.lang.Close() + obj.cancel() + obj.wgRun.Wait() + err := obj.lang.Cleanup() + err = errwrap.Append(err, obj.reterr) // from obj.lang.Run obj.lang = nil // clear it to avoid double closing return errwrap.Wrapf(err, "can't close the lang") // nil passthrough } @@ -520,7 +539,7 @@ func (obj *GAPI) Next() chan gapi.Next { startChan := make(chan struct{}) // start signal close(startChan) // kick it off! - streamChan := make(chan error) + streamChan := make(<-chan error) //defer obj.LangClose() // close any old lang var ok bool @@ -545,6 +564,7 @@ func (obj *GAPI) Next() chan gapi.Next { obj.data.Logf("generating new graph...") // skip this to pass through the err if present + // XXX: redo this old garbage code if langSwap && err == nil { obj.data.Logf("swap!") // run up to these three but fail on err diff --git a/lang/interfaces/ast.go b/lang/interfaces/ast.go index 0bf9fe2f0d..ddbb2efca0 100644 --- a/lang/interfaces/ast.go +++ b/lang/interfaces/ast.go @@ -72,12 +72,14 @@ type Stmt interface { // returns the collection to the caller. Unify() ([]Invariant, error) - // Graph returns the reactive function graph expressed by this node. + // Graph returns the reactive function graph expressed by this node. It + // takes in the environment of any functions in scope. Graph() (*pgraph.Graph, error) // Output returns the output that this "program" produces. This output - // is what is used to build the output graph. - Output() (*Output, error) + // is what is used to build the output graph. It requires the input + // table of values that are used to populate each function. + Output(map[Func]types.Value) (*Output, error) } // Expr represents an expression in the language. Expr implementations must have @@ -106,7 +108,7 @@ type Expr interface { Ordering(map[string]Node) (*pgraph.Graph, map[Node]string, error) // SetScope sets the scope here and propagates it downwards. - SetScope(*Scope) error + SetScope(*Scope, map[string]Expr) error // SetType sets the type definitively, and errors if it is incompatible. SetType(*types.Type) error @@ -120,11 +122,10 @@ type Expr interface { // returns the collection to the caller. Unify() ([]Invariant, error) - // Graph returns the reactive function graph expressed by this node. - Graph() (*pgraph.Graph, error) - - // Func returns a function that represents this reactively. - Func() (Func, error) + // Graph returns the reactive function graph expressed by this node. It + // takes in the environment of any functions in scope. It also returns + // the function for this node. + Graph(env map[string]Func) (*pgraph.Graph, Func, error) // SetValue stores the result of the last computation of this expression // node. @@ -134,6 +135,15 @@ type Expr interface { Value() (types.Value, error) } +// ScopeGrapher adds a method to turn an AST (Expr or Stmt) into a graph so that +// we can debug the SetScope compilation phase. +type ScopeGrapher interface { + Node + + // ScopeGraph adds nodes and vertices to the supplied graph. + ScopeGraph(g *pgraph.Graph) +} + // Data provides some data to the node that could be useful during its lifetime. type Data struct { // Fs represents a handle to the filesystem that we're running on. This @@ -228,9 +238,6 @@ type Scope struct { Variables map[string]Expr Functions map[string]Expr // the Expr will usually be an *ExprFunc Classes map[string]Stmt - // TODO: It is easier to shift a list, but let's use a map for Indexes - // for now in case we ever need holes... - Indexes map[int][]Expr // TODO: use [][]Expr instead? Chain []Node // chain of previously seen node's } @@ -242,7 +249,6 @@ func EmptyScope() *Scope { Variables: make(map[string]Expr), Functions: make(map[string]Expr), Classes: make(map[string]Stmt), - Indexes: make(map[int][]Expr), Chain: []Node{}, } } @@ -259,9 +265,6 @@ func (obj *Scope) InitScope() { if obj.Classes == nil { obj.Classes = make(map[string]Stmt) } - if obj.Indexes == nil { - obj.Indexes = make(map[int][]Expr) - } if obj.Chain == nil { obj.Chain = []Node{} } @@ -275,7 +278,6 @@ func (obj *Scope) Copy() *Scope { variables := make(map[string]Expr) functions := make(map[string]Expr) classes := make(map[string]Stmt) - indexes := make(map[int][]Expr) chain := []Node{} if obj != nil { // allow copying nil scopes obj.InitScope() // safety @@ -288,13 +290,6 @@ func (obj *Scope) Copy() *Scope { for k, v := range obj.Classes { // copy classes[k] = v // we don't copy the StmtClass! } - for k, v := range obj.Indexes { // copy - ixs := []Expr{} - for _, x := range v { - ixs = append(ixs, x) // we don't copy the expr's! - } - indexes[k] = ixs - } for _, x := range obj.Chain { // copy chain = append(chain, x) // we don't copy the Stmt pointer! } @@ -303,7 +298,6 @@ func (obj *Scope) Copy() *Scope { Variables: variables, Functions: functions, Classes: classes, - Indexes: indexes, Chain: chain, } } @@ -357,9 +351,6 @@ func (obj *Scope) Merge(scope *Scope) error { obj.Classes[name] = scope.Classes[name] } - // FIXME: should we merge or overwrite? (I think this isn't even used) - obj.Indexes = scope.Indexes // overwrite without error - return err } @@ -375,80 +366,12 @@ func (obj *Scope) IsEmpty() bool { if len(obj.Functions) > 0 { return false } - if len(obj.Indexes) > 0 { // FIXME: should we check each one? (unused?) - return false - } if len(obj.Classes) > 0 { return false } return true } -// MaxIndexes returns the maximum index of Indexes stored in the scope. If it is -// empty then -1 is returned. -func (obj *Scope) MaxIndexes() int { - obj.InitScope() // safety - max := -1 - for k := range obj.Indexes { - if k > max { - max = k - } - } - return max -} - -// PushIndexes adds a list of expressions at the zeroth index in Indexes after -// firsh pushing everyone else over by one. If you pass in nil input this may -// panic! -func (obj *Scope) PushIndexes(exprs []Expr) { - if exprs == nil { - // TODO: is this the right thing to do? - panic("unexpected nil input") - } - obj.InitScope() // safety - max := obj.MaxIndexes() - for i := max; i >= 0; i-- { // reverse order - indexes, exists := obj.Indexes[i] - if !exists { - continue - } - delete(obj.Indexes, i) - obj.Indexes[i+1] = indexes // push it - } - - if obj.Indexes == nil { // in case we weren't initialized yet - obj.Indexes = make(map[int][]Expr) - } - obj.Indexes[0] = exprs // usually the list of Args in ExprCall -} - -// PullIndexes takes a list of expressions from the zeroth index in Indexes and -// then pulls everyone over by one. The returned value is only valid if one was -// found at the zeroth index. The returned boolean will be true if it exists. -func (obj *Scope) PullIndexes() ([]Expr, bool) { - obj.InitScope() // safety - if obj.Indexes == nil { // in case we weren't initialized yet - obj.Indexes = make(map[int][]Expr) - } - - indexes, exists := obj.Indexes[0] // save for later - - max := obj.MaxIndexes() - for i := 0; i <= max; i++ { - ixs, exists := obj.Indexes[i] - if !exists { - continue - } - delete(obj.Indexes, i) - if i == 0 { // zero falls off - continue - } - obj.Indexes[i-1] = ixs - } - - return indexes, exists -} - // Arg represents a name identifier for a func or class argument declaration and // is sometimes accompanied by a type. This does not satisfy the Expr interface. type Arg struct { diff --git a/lang/interfaces/func.go b/lang/interfaces/func.go index 6f381ae571..62acda0a26 100644 --- a/lang/interfaces/func.go +++ b/lang/interfaces/func.go @@ -24,6 +24,7 @@ import ( "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/pgraph" ) // FuncSig is the simple signature that is used throughout our implementations. @@ -45,8 +46,20 @@ type Info struct { type Init struct { Hostname string // uuid for the host //Noop bool - Input chan types.Value // Engine will close `input` chan - Output chan types.Value // Stream must close `output` chan + + // Input is where a chan (stream) of values will get sent to this node. + // The engine will close this `input` chan. + Input chan types.Value + + // Output is the chan (stream) of values to get sent out from this node. + // The Stream function must close this `output` chan. + Output chan types.Value + + // Txn provides a transaction API that can be used to modify the + // function graph while it is "running". This should not be used by most + // nodes, and when it is used, it should be used carefully. + Txn Txn + // TODO: should we pass in a *Scope here for functions like template() ? World engine.World Debug bool @@ -118,11 +131,19 @@ type PolyFunc interface { // another way: the Expr input is the ExprFunc, not the ExprCall. Unify(Expr) ([]Invariant, error) - // Build takes the known type signature for this function and finalizes - // this structure so that it is now determined, and ready to function as - // a normal function would. (The normal methods in the Func interface - // are all that should be needed or used after this point.) - Build(*types.Type) error // then, you can get argNames from Info() + // Build takes the known or unified type signature for this function and + // finalizes this structure so that it is now determined, and ready to + // function as a normal function would. (The normal methods in the Func + // interface are all that should be needed or used after this point.) + // Of note, the names of the specific input args shouldn't matter as + // long as they are unique. Their position doesn't matter. This is so + // that unification can use "arg0", "arg1", "argN"... if they can't be + // determined statically. Build can transform them into it's desired + // form, and must return the type (with the correct arg names) that it + // will use. These are used when constructing the function graphs. This + // means that when this is called from SetType, it can set the correct + // type arg names, and this will also match what's in function Info(). + Build(*types.Type) (*types.Type, error) } // OldPolyFunc is an interface for functions which are statically polymorphic. @@ -150,11 +171,19 @@ type OldPolyFunc interface { // want to convert easily. Polymorphisms(*types.Type, []types.Value) ([]*types.Type, error) - // Build takes the known type signature for this function and finalizes - // this structure so that it is now determined, and ready to function as - // a normal function would. (The normal methods in the Func interface - // are all that should be needed or used after this point.) - Build(*types.Type) error // then, you can get argNames from Info() + // Build takes the known or unified type signature for this function and + // finalizes this structure so that it is now determined, and ready to + // function as a normal function would. (The normal methods in the Func + // interface are all that should be needed or used after this point.) + // Of note, the names of the specific input args shouldn't matter as + // long as they are unique. Their position doesn't matter. This is so + // that unification can use "arg0", "arg1", "argN"... if they can't be + // determined statically. Build can transform them into it's desired + // form, and must return the type (with the correct arg names) that it + // will use. These are used when constructing the function graphs. This + // means that when this is called from SetType, it can set the correct + // type arg names, and this will also match what's in function Info(). + Build(*types.Type) (*types.Type, error) } // NamedArgsFunc is a function that uses non-standard function arg names. If you @@ -214,3 +243,93 @@ type FuncEdge struct { func (obj *FuncEdge) String() string { return strings.Join(obj.Args, ", ") } + +// GraphAPI is a subset of the available graph operations that are possible on a +// pgraph that is used for storing functions. The minimum subset are those which +// are needed for implementing the Txn interface. +type GraphAPI interface { + AddVertex(Func) error + AddEdge(Func, Func, *FuncEdge) error + DeleteVertex(Func) error + DeleteEdge(*FuncEdge) error + //AddGraph(*pgraph.Graph) error + + //Adjacency() map[Func]map[Func]*FuncEdge + HasVertex(Func) bool + FindEdge(Func, Func) *FuncEdge + LookupEdge(*FuncEdge) (Func, Func, bool) +} + +// Txn is the interface that the engine graph API makes available so that +// functions can modify the function graph dynamically while it is "running". +// This could be implemented in one of two methods. +// +// Method 1: Have a pair of graph Lock and Unlock methods. Queue up the work to +// do and when we "commit" the transaction, we're just queuing up the work to do +// and then we run it all surrounded by the lock. +// +// Method 2: It's possible that we might eventually be able to actually modify +// the running graph without even causing it to pause at all. In this scenario, +// the "commit" would just directly perform those operations without even using +// the Lock and Unlock mutex operations. This is why we don't expose those in +// the API. It's also safer because someone can't forget to run Unlock which +// would block the whole code base. +type Txn interface { + // AddVertex adds a vertex to the running graph. The operation will get + // completed when Commit is run. + AddVertex(Func) Txn + + // AddEdge adds an edge to the running graph. The operation will get + // completed when Commit is run. + AddEdge(Func, Func, *FuncEdge) Txn + + // DeleteVertex removes a vertex from the running graph. The operation + // will get completed when Commit is run. + DeleteVertex(Func) Txn + + // DeleteEdge removes an edge from the running graph. It removes the + // edge that is found between the two input vertices. The operation will + // get completed when Commit is run. The edge is part of the signature + // so that it is both symmetrical with AddEdge, and also easier to + // reverse in theory. + // NOTE: This is not supported since there's no sane Reverse with GC. + // XXX: Add this in but just don't let it be reversible? + //DeleteEdge(Func, Func, *FuncEdge) Txn + + // AddGraph adds a graph to the running graph. The operation will get + // completed when Commit is run. This function panics if your graph + // contains vertices that are not of type interfaces.Func or if your + // edges are not of type *interfaces.FuncEdge. + AddGraph(*pgraph.Graph) Txn + + // Commit runs the pending transaction. + Commit() error + + // Clear erases any pending transactions that weren't committed yet. + Clear() + + // Reverse runs the reverse commit of the last successful operation to + // Commit. AddVertex is reversed by DeleteVertex, and vice-versa, and + // the same for AddEdge and DeleteEdge. Keep in mind that if AddEdge is + // called with either vertex not already part of the graph, it will + // implicitly add them, but the Reverse operation will not necessarily + // know that. As a result, it's recommended to not perform operations + // that have implicit Adds or Deletes. Notwithstanding the above, the + // initial Txn implementation can and does try to track these changes + // so that it can correctly reverse them, but this is not guaranteed by + // API, and it could contain bugs. + Reverse() error + + // Erase removes the historical information that Reverse would run after + // Commit. + Erase() + + // Free releases the wait group that was used to lock around this Txn if + // needed. It should get called when we're done with any Txn. + Free() + + // Copy returns a new child Txn that has the same handles, but a + // separate state. This allows you to do an Add*/Commit/Reverse that + // isn't affected by a different user of this transaction. + Copy() Txn +} diff --git a/lang/interfaces/structs.go b/lang/interfaces/structs.go index 93721ecad0..3db45da8bb 100644 --- a/lang/interfaces/structs.go +++ b/lang/interfaces/structs.go @@ -76,7 +76,7 @@ func (obj *ExprAny) Ordering(produces map[string]Node) (*pgraph.Graph, map[Node] // SetScope does nothing for this struct, because it has no child nodes, and it // does not need to know about the parent scope. -func (obj *ExprAny) SetScope(*Scope) error { return nil } +func (obj *ExprAny) SetScope(*Scope, map[string]Expr) error { return nil } // SetType is used to set the type of this expression once it is known. This // usually happens during type unification, but it can also happen during @@ -128,6 +128,16 @@ func (obj *ExprAny) Unify() ([]Invariant, error) { return invariants, nil } +// Func returns the reactive stream of values that this expression produces. +func (obj *ExprAny) Func() (Func, error) { + // // XXX: this could be a list too, so improve this code or improve the subgraph code... + // return &structs.ConstFunc{ + // Value: obj.V, + // } + + return nil, fmt.Errorf("programming error using ExprAny") // this should not be called +} + // Graph returns the reactive function graph which is expressed by this node. It // includes any vertices produced by this node, and the appropriate edges to any // vertices that are produced by its children. Nodes which fulfill the Expr @@ -135,23 +145,17 @@ func (obj *ExprAny) Unify() ([]Invariant, error) { // that fulfill the Stmt interface do not produces vertices, where as their // children might. This returns a graph with a single vertex (itself) in it, and // the edges from all of the child graphs to this. -func (obj *ExprAny) Graph() (*pgraph.Graph, error) { +func (obj *ExprAny) Graph(env map[string]Func) (*pgraph.Graph, Func, error) { graph, err := pgraph.NewGraph("any") if err != nil { - return nil, err + return nil, nil, err } - graph.AddVertex(obj) - return graph, nil -} - -// Func returns the reactive stream of values that this expression produces. -func (obj *ExprAny) Func() (Func, error) { - // // XXX: this could be a list too, so improve this code or improve the subgraph code... - // return &structs.ConstFunc{ - // Value: obj.V, - // } - - return nil, fmt.Errorf("programming error using ExprAny") // this should not be called + function, err := obj.Func() + if err != nil { + return nil, nil, err + } + graph.AddVertex(function) + return graph, function, nil } // SetValue here is a no-op, because algorithmically when this is called from @@ -182,3 +186,8 @@ func (obj *ExprAny) Value() (types.Value, error) { } return obj.V, nil } + +// ScopeGraph adds nodes and vertices to the supplied graph. +func (obj *ExprAny) ScopeGraph(g *pgraph.Graph) { + g.AddVertex(obj) +} diff --git a/lang/interpret/interpret.go b/lang/interpret/interpret.go index e0495117ec..2c9e307a5f 100644 --- a/lang/interpret/interpret.go +++ b/lang/interpret/interpret.go @@ -25,16 +25,17 @@ import ( "github.com/purpleidea/mgmt/engine" engineUtil "github.com/purpleidea/mgmt/engine/util" "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" "github.com/purpleidea/mgmt/pgraph" "github.com/purpleidea/mgmt/util/errwrap" ) -// Interpret runs the program and causes a graph generation as a side effect. -// You should not run this on the AST if you haven't previously run the function -// graph engine so that output values have been produced! Type unification is -// another important aspect which needs to have been completed. -func Interpret(ast interfaces.Stmt) (*pgraph.Graph, error) { - output, err := ast.Output() // contains resList, edgeList, etc... +// Interpret runs the program and outputs a generated resource graph. It +// requires an AST, and the table of values required to populate that AST. Type +// unification, and earlier steps should obviously be run first so that you can +// actually get a useful resource graph out of this instead of an error! +func Interpret(ast interfaces.Stmt, table map[interfaces.Func]types.Value) (*pgraph.Graph, error) { + output, err := ast.Output(table) // contains resList, edgeList, etc... if err != nil { return nil, err } diff --git a/lang/interpret_test.go b/lang/interpret_test.go index 2cdc3c3f1c..26189bda12 100644 --- a/lang/interpret_test.go +++ b/lang/interpret_test.go @@ -21,21 +21,21 @@ package lang import ( "bytes" + "context" "fmt" "io/ioutil" "os" "path/filepath" "sort" "strings" + "sync" "testing" "time" - "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/engine/graph/autoedge" - "github.com/purpleidea/mgmt/engine/resources" "github.com/purpleidea/mgmt/etcd" "github.com/purpleidea/mgmt/lang/ast" - "github.com/purpleidea/mgmt/lang/funcs" + "github.com/purpleidea/mgmt/lang/funcs/dage" "github.com/purpleidea/mgmt/lang/funcs/vars" "github.com/purpleidea/mgmt/lang/inputs" "github.com/purpleidea/mgmt/lang/interfaces" @@ -55,503 +55,6 @@ const ( runGraphviz = false // run graphviz in tests? ) -func vertexAstCmpFn(v1, v2 pgraph.Vertex) (bool, error) { - //fmt.Printf("V1: %T %+v\n", v1, v1) - //node := v1.(*funcs.Node) - //fmt.Printf("node: %T %+v\n", node, node) - //fmt.Printf("V2: %T %+v\n", v2, v2) - if v1.String() == "" || v2.String() == "" { - return false, fmt.Errorf("oops, empty vertex") - } - return v1.String() == v2.String(), nil -} - -func edgeAstCmpFn(e1, e2 pgraph.Edge) (bool, error) { - if e1.String() == "" || e2.String() == "" { - return false, fmt.Errorf("oops, empty edge") - } - return e1.String() == e2.String(), nil -} - -type vtex string - -func (obj *vtex) String() string { - return string(*obj) -} - -type edge string - -func (obj *edge) String() string { - return string(*obj) -} - -func TestAstFunc0(t *testing.T) { - scope := &interfaces.Scope{ // global scope - Variables: map[string]interfaces.Expr{ - "hello": &ast.ExprStr{V: "world"}, - "answer": &ast.ExprInt{V: 42}, - }, - // all the built-in top-level, core functions enter here... - Functions: ast.FuncPrefixToFunctionsScope(""), // runs funcs.LookupPrefix - } - - type test struct { // an individual test - name string - code string - fail bool - scope *interfaces.Scope - graph *pgraph.Graph - } - testCases := []test{} - - { - graph, _ := pgraph.NewGraph("g") - testCases = append(testCases, test{ // 0 - "nil", - ``, - false, - nil, - graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - testCases = append(testCases, test{ - name: "scope only", - code: ``, - fail: false, - scope: scope, // use the scope defined above - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - // empty graph at the moment, because they're all unused! - //v1, v2 := vtex("int(42)"), vtex("var(x)") - //e1 := edge("var:x") - //graph.AddVertex(&v1, &v2) - //graph.AddEdge(&v1, &v2, &e1) - testCases = append(testCases, test{ - name: "two vars", - code: ` - $x = 42 - $y = $x - `, - // TODO: this should fail with an unused variable error! - fail: false, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - testCases = append(testCases, test{ - name: "self-referential vars", - code: ` - $x = $y - $y = $x - `, - fail: true, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2, v3, v4, v5 := vtex("int(42)"), vtex("var(a)"), vtex("var(b)"), vtex("var(c)"), vtex(`str("t")`) - e1, e2, e3 := edge("var:a"), edge("var:b"), edge("var:c") - graph.AddVertex(&v1, &v2, &v3, &v4, &v5) - graph.AddEdge(&v1, &v2, &e1) - graph.AddEdge(&v2, &v3, &e2) - graph.AddEdge(&v3, &v4, &e3) - testCases = append(testCases, test{ - name: "chained vars", - code: ` - test "t" { - int64ptr => $c, - } - $c = $b - $b = $a - $a = 42 - `, - fail: false, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2 := vtex("bool(true)"), vtex("var(b)") - graph.AddVertex(&v1, &v2) - e1 := edge("var:b") - graph.AddEdge(&v1, &v2, &e1) - testCases = append(testCases, test{ - name: "simple bool", - code: ` - if $b { - } - $b = true - `, - fail: false, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2, v3, v4, v5 := vtex(`str("t")`), vtex(`str("+")`), vtex("int(42)"), vtex("int(13)"), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, funcs.OperatorFuncName)) - graph.AddVertex(&v1, &v2, &v3, &v4, &v5) - e1, e2, e3 := edge("op"), edge("a"), edge("b") - graph.AddEdge(&v2, &v5, &e1) - graph.AddEdge(&v3, &v5, &e2) - graph.AddEdge(&v4, &v5, &e3) - testCases = append(testCases, test{ - name: "simple operator", - code: ` - test "t" { - int64ptr => 42 + 13, - } - `, - fail: false, - scope: scope, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2, v3 := vtex(`str("t")`), vtex(`str("-")`), vtex(`str("+")`) - v4, v5, v6 := vtex("int(42)"), vtex("int(13)"), vtex("int(99)") - v7 := vtex(fmt.Sprintf(`call:%s(str("+"), int(42), int(13))`, funcs.OperatorFuncName)) - v8 := vtex(fmt.Sprintf(`call:%s(str("-"), call:%s(str("+"), int(42), int(13)), int(99))`, funcs.OperatorFuncName, funcs.OperatorFuncName)) - - graph.AddVertex(&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8) - e1, e2, e3 := edge("op"), edge("a"), edge("b") - graph.AddEdge(&v3, &v7, &e1) - graph.AddEdge(&v4, &v7, &e2) - graph.AddEdge(&v5, &v7, &e3) - - e4, e5, e6 := edge("op"), edge("a"), edge("b") - graph.AddEdge(&v2, &v8, &e4) - graph.AddEdge(&v7, &v8, &e5) - graph.AddEdge(&v6, &v8, &e6) - testCases = append(testCases, test{ - name: "simple operators", - code: ` - test "t" { - int64ptr => 42 + 13 - 99, - } - `, - fail: false, - scope: scope, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2 := vtex("bool(true)"), vtex(`str("t")`) - v3, v4 := vtex("int(13)"), vtex("int(42)") - v5, v6 := vtex("var(i)"), vtex("var(x)") - v7, v8 := vtex(`str("+")`), vtex(fmt.Sprintf(`call:%s(str("+"), int(42), var(i))`, funcs.OperatorFuncName)) - - e1, e2, e3, e4, e5 := edge("op"), edge("a"), edge("b"), edge("var:i"), edge("var:x") - graph.AddVertex(&v1, &v2, &v3, &v4, &v5, &v6, &v7, &v8) - graph.AddEdge(&v3, &v5, &e4) - - graph.AddEdge(&v7, &v8, &e1) - graph.AddEdge(&v4, &v8, &e2) - graph.AddEdge(&v5, &v8, &e3) - - graph.AddEdge(&v8, &v6, &e5) - testCases = append(testCases, test{ - name: "nested resource and scoped var", - code: ` - if true { - test "t" { - int64ptr => $x, - } - $x = 42 + $i - } - $i = 13 - `, - fail: false, - scope: scope, - graph: graph, - }) - } - { - testCases = append(testCases, test{ - name: "out of scope error", - code: ` - # should be out of scope, and a compile error! - if $b { - } - if true { - $b = true - } - `, - fail: true, - }) - } - { - testCases = append(testCases, test{ - name: "variable re-declaration error", - code: ` - # this should fail b/c of variable re-declaration - $x = "hello" - $x = "world" # woops - `, - fail: true, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2, v3 := vtex(`str("hello")`), vtex(`str("world")`), vtex("bool(true)") - v4, v5 := vtex("var(x)"), vtex(`str("t")`) - - graph.AddVertex(&v1, &v3, &v4, &v5) - _ = v2 // v2 is not used because it's shadowed! - e1 := edge("var:x") - // only one edge! (cool) - graph.AddEdge(&v1, &v4, &e1) - - testCases = append(testCases, test{ - name: "variable shadowing", - code: ` - # this should be okay, because var is shadowed - $x = "hello" - if true { - $x = "world" # shadowed - } - test "t" { - stringptr => $x, - } - `, - fail: false, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - v1, v2, v3 := vtex(`str("hello")`), vtex(`str("world")`), vtex("bool(true)") - v4, v5 := vtex("var(x)"), vtex(`str("t")`) - - graph.AddVertex(&v2, &v3, &v4, &v5) - _ = v1 // v1 is not used because it's shadowed! - e1 := edge("var:x") - // only one edge! (cool) - graph.AddEdge(&v2, &v4, &e1) - - testCases = append(testCases, test{ - name: "variable shadowing inner", - code: ` - # this should be okay, because var is shadowed - $x = "hello" - if true { - $x = "world" # shadowed - test "t" { - stringptr => $x, - } - } - `, - fail: false, - graph: graph, - }) - } - // // FIXME: blocked by: https://github.com/purpleidea/mgmt/issues/199 - //{ - // graph, _ := pgraph.NewGraph("g") - // v0 := vtex("bool(true)") - // v1, v2 := vtex(`str("hello")`), vtex(`str("world")`) - // v3, v4 := vtex("var(x)"), vtex("var(x)") // different vertices! - // v5, v6 := vtex(`str("t1")`), vtex(`str("t2")`) - // - // graph.AddVertex(&v0, &v1, &v2, &v3, &v4, &v5, &v6) - // e1, e2 := edge("var:x"), edge("var:x") - // graph.AddEdge(&v1, &v3, &e1) - // graph.AddEdge(&v2, &v4, &e2) - // - // testCases = append(testCases, test{ - // name: "variable shadowing both", - // code: ` - // # this should be okay, because var is shadowed - // $x = "hello" - // if true { - // $x = "world" # shadowed - // test "t2" { - // stringptr => $x, - // } - // } - // test "t1" { - // stringptr => $x, - // } - // `, - // fail: false, - // graph: graph, - // }) - //} - // // FIXME: blocked by: https://github.com/purpleidea/mgmt/issues/199 - //{ - // graph, _ := pgraph.NewGraph("g") - // v1, v2 := vtex(`str("cowsay")`), vtex(`str("cowsay")`) - // v3, v4 := vtex(`str("installed)`), vtex(`str("newest")`) - // - // graph.AddVertex(&v1, &v2, &v3, &v4) - // - // testCases = append(testCases, test{ - // name: "duplicate resource", - // code: ` - // # these two are allowed because they are compatible - // pkg "cowsay" { - // state => "installed", - // } - // pkg "cowsay" { - // state => "newest", - // } - // `, - // fail: false, - // graph: graph, - // }) - //} - { - testCases = append(testCases, test{ - name: "variable re-declaration and type change error", - code: ` - # this should fail b/c of variable re-declaration - $x = "wow" - $x = 99 # woops, but also a change of type :P - `, - fail: true, - }) - } - - names := []string{} - for index, tc := range testCases { // run all the tests - if tc.name == "" { - t.Errorf("test #%d: not named", index) - continue - } - if util.StrInList(tc.name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) - continue - } - names = append(names, tc.name) - - //if index != 3 { // hack to run a subset (useful for debugging) - //if tc.name != "simple operators" { - // continue - //} - - t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { - name, code, fail, scope, exp := tc.name, tc.code, tc.fail, tc.scope, tc.graph - - t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) - str := strings.NewReader(code) - xast, err := parser.LexParse(str) - if err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: lex/parse failed with: %+v", index, err) - return - } - t.Logf("test #%d: AST: %+v", index, xast) - - data := &interfaces.Data{ - // TODO: add missing fields here if/when needed - StrInterpolater: interpolate.StrInterpolate, - - Debug: testing.Verbose(), // set via the -test.v flag to `go test` - Logf: func(format string, v ...interface{}) { - t.Logf("ast: "+format, v...) - }, - } - // some of this might happen *after* interpolate in SetScope or Unify... - if err := xast.Init(data); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not init and validate AST: %+v", index, err) - return - } - - iast, err := xast.Interpolate() - if err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: interpolate failed with: %+v", index, err) - return - } - - // propagate the scope down through the AST... - err = iast.SetScope(scope) - if !fail && err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not set scope: %+v", index, err) - return - } - if fail && err != nil { - return // fail happened during set scope, don't run unification! - } - - // apply type unification - logf := func(format string, v ...interface{}) { - t.Logf(fmt.Sprintf("test #%d", index)+": unification: "+format, v...) - } - unifier := &unification.Unifier{ - AST: iast, - Solver: unification.SimpleInvariantSolverLogger(logf), - Debug: testing.Verbose(), - Logf: logf, - } - err = unifier.Unify() - if !fail && err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not unify types: %+v", index, err) - return - } - // maybe it will fail during graph below instead? - //if fail && err == nil { - // t.Errorf("test #%d: FAIL", index) - // t.Errorf("test #%d: unification passed, expected fail", index) - // continue - //} - if fail && err != nil { - return // fail happened during unification, don't run Graph! - } - - // build the function graph - graph, err := iast.Graph() - - if !fail && err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: functions failed with: %+v", index, err) - return - } - if fail && err == nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: functions passed, expected fail", index) - return - } - - if fail { // can't process graph if it's nil - // TODO: match against expected error - t.Logf("test #%d: error: %+v", index, err) - return - } - - t.Logf("test #%d: graph: %+v", index, graph) - // TODO: improve: https://github.com/purpleidea/mgmt/issues/199 - if err := graph.GraphCmp(exp, vertexAstCmpFn, edgeAstCmpFn); err != nil { - t.Errorf("test #%d: FAIL\n\n", index) - t.Logf("test #%d: actual (g1): %v%s\n\n", index, graph, fullPrint(graph)) - t.Logf("test #%d: expected (g2): %v%s\n\n", index, exp, fullPrint(exp)) - t.Errorf("test #%d: cmp error:\n%v", index, err) - return - } - - for i, v := range graph.Vertices() { - t.Logf("test #%d: vertex(%d): %+v", index, i, v) - } - for v1 := range graph.Adjacency() { - for v2, e := range graph.Adjacency()[v1] { - t.Logf("test #%d: edge(%+v): %+v -> %+v", index, e, v1, v2) - } - } - }) - } -} - // TestAstFunc1 is a more advanced version which pulls code from physical dirs. func TestAstFunc1(t *testing.T) { const magicError = "# err: " @@ -918,7 +421,6 @@ func TestAstFunc1(t *testing.T) { // build the function graph graph, err := iast.Graph() - if (!fail || !failGraph) && err != nil { t.Errorf("test #%d: FAIL", index) t.Errorf("test #%d: functions failed with: %+v", index, err) @@ -1384,6 +886,31 @@ func TestAstFunc2(t *testing.T) { return } + if runGraphviz { + t.Logf("test #%d: Running graphviz after setScope...", index) + + // build a graph of the AST, to make sure everything is connected properly + graph, err := pgraph.NewGraph("setScope") + if err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: could not create setScope graph: %+v", index, err) + return + } + ast, ok := iast.(interfaces.ScopeGrapher) + if !ok { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: can't graph scope", index) + return + } + ast.ScopeGraph(graph) + + if err := graph.ExecGraphviz("/tmp/set-scope.dot"); err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: writing graph failed: %+v", index, err) + return + } + } + // apply type unification xlogf := func(format string, v ...interface{}) { logf("unification: "+format, v...) @@ -1418,7 +945,6 @@ func TestAstFunc2(t *testing.T) { // build the function graph graph, err := iast.Graph() - if (!fail || !failGraph) && err != nil { t.Errorf("test #%d: FAIL", index) t.Errorf("test #%d: functions failed with: %+v", index, err) @@ -1441,9 +967,9 @@ func TestAstFunc2(t *testing.T) { } if graph.NumVertices() == 0 { // no funcs to load! - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: function graph is empty", index) - return + //t.Errorf("test #%d: FAIL", index) + t.Logf("test #%d: function graph is empty", index) + //return // let's test the engine on empty } t.Logf("test #%d: graph: %s", index, graph) @@ -1465,81 +991,115 @@ func TestAstFunc2(t *testing.T) { } } - // XXX: temporary compatibility mapping for now... - // XXX: this could be a helper function eventually... - //// map the graph from interfaces.Expr to interfaces.Func - //mapExprFunc := make(map[interfaces.Expr]interfaces.Func) - //for v1, x := range graph.Adjacency() { - // v1, ok := v1.(interfaces.Expr) - // if !ok { - // panic("programming error") - // } - // if _, exists := mapExprFunc[v1]; !exists { - // var err error - // mapExprFunc[v1], err = v1.Func() - // if err != nil { - // panic("programming error") - // } - // } - // //funcs.AddVertex(v1) - // for v2 := range x { - // v2, ok := v2.(interfaces.Expr) - // if !ok { - // panic("programming error") - // } - // if _, exists := mapExprFunc[v2]; !exists { - // var err error - // mapExprFunc[v2], err = v2.Func() - // if err != nil { - // panic("programming error") - // } - // - // } - // //funcs.AddEdge(v1, v2, edge) - // } - //} - // run the function engine once to get some real output - funcs := &funcs.Engine{ - Graph: graph, // not the same as the output graph! + funcs := &dage.Engine{ + Name: "test", Hostname: "", // NOTE: empty b/c not used World: world, // used partially in some tests Debug: testing.Verbose(), // set via the -test.v flag to `go test` Logf: func(format string, v ...interface{}) { logf("funcs: "+format, v...) }, - Glitch: false, // FIXME: verify this functionality is perfect! } logf("function engine initializing...") - if err := funcs.Init(); err != nil { + if err := funcs.Setup(); err != nil { t.Errorf("test #%d: FAIL", index) t.Errorf("test #%d: init error with func engine: %+v", index, err) return } + defer funcs.Cleanup() - logf("function engine validating...") - if err := funcs.Validate(); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: validate error with func engine: %+v", index, err) - return - } + // XXX: can we type check things somehow? + //logf("function engine validating...") + //if err := funcs.Validate(); err != nil { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: validate error with func engine: %+v", index, err) + // return + //} logf("function engine starting...") - // On failure, we expect the caller to run Close() to shutdown all of - // the currently initialized (and running) funcs... This is needed if - // we successfully ran `Run` but isn't needed only for Init/Validate. - if err := funcs.Run(); err != nil { + wg := &sync.WaitGroup{} + defer wg.Wait() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + wg.Add(1) + go func() { + defer wg.Done() + if err := funcs.Run(ctx); err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: run error with func engine: %+v", index, err) + return + } + }() + + //wg.Add(1) + //go func() { // XXX: debugging + // defer wg.Done() + // for { + // select { + // case <-time.After(100 * time.Millisecond): // blocked functions + // t.Logf("test #%d: graphviz...", index) + // funcs.Graphviz("") // log to /tmp/... + // + // case <-ctx.Done(): + // return + // } + // } + //}() + + <-funcs.Started() // wait for startup (will not block forever) + + // Sanity checks for graph size. + if count := funcs.NumVertices(); count != 0 { t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: run error with func engine: %+v", index, err) + t.Errorf("test #%d: expected empty graph on start, got %d vertices", index, count) + } + defer func() { + if count := funcs.NumVertices(); count != 0 { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: expected empty graph on exit, got %d vertices", index, count) + } + }() + defer wg.Wait() + defer cancel() + + txn := funcs.Txn() + defer txn.Free() // remember to call Free() + txn.AddGraph(graph) + if err := txn.Commit(); err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: run error with initial commit: %+v", index, err) return } - // TODO: cleanup before we print any test failures... - defer funcs.Close() // cleanup + defer txn.Reverse() // should remove everything we added + + isEmpty := make(chan struct{}) + if graph.NumVertices() == 0 { // no funcs to load! + close(isEmpty) + } // wait for some activity logf("stream...") stream := funcs.Stream() + //select { + //case err, ok := <-stream: + // if !ok { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream closed", index) + // return + // } + // if err != nil { + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream errored: %+v", index, err) + // return + // } + // + //case <-time.After(60 * time.Second): // blocked functions + // t.Errorf("test #%d: FAIL", index) + // t.Errorf("test #%d: stream timeout", index) + // return + //} // sometimes the <-stream seems to constantly (or for a // long time?) win the races against the <-time.After(), @@ -1559,13 +1119,17 @@ func TestAstFunc2(t *testing.T) { t.Errorf("test #%d: stream errored: %+v", index, err) return } - t.Logf("test #%d: stream...", index) + t.Logf("test #%d: got stream event!", index) max-- if max == 0 { break Loop } - case <-time.After(3 * time.Second): // blocked functions + case <-isEmpty: + break Loop + + case <-time.After(10 * time.Second): // blocked functions + t.Errorf("test #%d: unblocking because no event was sent by the function engine for a while", index) break Loop case <-time.After(60 * time.Second): // blocked functions @@ -1575,51 +1139,12 @@ func TestAstFunc2(t *testing.T) { } } + t.Logf("test #%d: %s", index, funcs.Stats()) + // run interpret! - table := funcs.Table() // map[pgraph.Vertex]types.Value - fn := func(n interfaces.Node) error { - expr, ok := n.(interfaces.Expr) - if !ok { - return nil - } - //f, exists := mapExprFunc[expr] - //if !exists { - // panic("programming error in mapExprFunc lookup") - //} - //val, exists := table[f] - //if !exists { - // fmt.Printf("XXX missing value in table is pointer: %p\n", f) - // return fmt.Errorf("missing value in table for: %s", f) - //} - - v, ok := expr.(pgraph.Vertex) - if !ok { - panic("programming error in interfaces.Expr -> pgraph.Vertex lookup") - } - val, exists := table[v] - if !exists { - // XXX: we have values in the AST which aren't need... - // XXX: confirmed with: time go test -race github.com/purpleidea/mgmt/lang/ -v -run TestAstFunc2/test_#42 (func-math1) for example. - fmt.Printf("XXX: missing value in table is pointer: %p\n", v) - return nil // XXX: workaround for now... - //return fmt.Errorf("missing value in table for: %s", v) - } - return expr.SetValue(val) // set the value - } - funcs.Lock() // XXX: apparently there are races between SetValue and reading obj.V values... - if err := iast.Apply(fn); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: apply failed with: %+v", index, err) - t.Errorf("test #%d: table:", index) - for k, v := range table { - t.Errorf("test #%d: table: key: %+v ; value: %+v", index, k, v) - } - funcs.Unlock() - return - } - funcs.Unlock() + table := funcs.Table() // map[interfaces.Func]types.Value - ograph, err := interpret.Interpret(iast) + ograph, err := interpret.Interpret(iast, table) if (!fail || !failInterpret) && err != nil { t.Errorf("test #%d: FAIL", index) t.Errorf("test #%d: interpret failed with: %+v", index, err) @@ -1709,246 +1234,3 @@ func TestAstFunc2(t *testing.T) { t.Skip("skipping all tests...") } } - -// TestAstInterpret0 should only be run in limited circumstances. Read the code -// comments below to see how it is run. -func TestAstInterpret0(t *testing.T) { - type test struct { // an individual test - name string - code string - fail bool - graph *pgraph.Graph - } - testCases := []test{} - - { - graph, _ := pgraph.NewGraph("g") - testCases = append(testCases, test{ // 0 - "nil", - ``, - false, - graph, - }) - } - { - testCases = append(testCases, test{ - name: "wrong res field type", - code: ` - test "t1" { - stringptr => 42, # int, not str - } - `, - fail: true, - }) - } - { - graph, _ := pgraph.NewGraph("g") - t1, _ := engine.NewNamedResource("test", "t1") - x := t1.(*resources.TestRes) - int64ptr := int64(42) - x.Int64Ptr = &int64ptr - str := "okay cool" - x.StringPtr = &str - int8ptr := int8(127) - int8ptrptr := &int8ptr - int8ptrptrptr := &int8ptrptr - x.Int8PtrPtrPtr = &int8ptrptrptr - graph.AddVertex(t1) - testCases = append(testCases, test{ - name: "resource with three pointer fields", - code: ` - test "t1" { - int64ptr => 42, - stringptr => "okay cool", - int8ptrptrptr => 127, # super nested - } - `, - fail: false, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - t1, _ := engine.NewNamedResource("test", "t1") - x := t1.(*resources.TestRes) - stringptr := "wow" - x.StringPtr = &stringptr - graph.AddVertex(t1) - testCases = append(testCases, test{ - name: "resource with simple string pointer field", - code: ` - test "t1" { - stringptr => "wow", - } - `, - graph: graph, - }) - } - { - // FIXME: add a better vertexCmpFn so we can compare send/recv! - graph, _ := pgraph.NewGraph("g") - t1, _ := engine.NewNamedResource("test", "t1") - { - x := t1.(*resources.TestRes) - int64Ptr := int64(42) - x.Int64Ptr = &int64Ptr - graph.AddVertex(t1) - } - t2, _ := engine.NewNamedResource("test", "t2") - { - x := t2.(*resources.TestRes) - int64Ptr := int64(13) - x.Int64Ptr = &int64Ptr - graph.AddVertex(t2) - } - edge := &engine.Edge{ - Name: fmt.Sprintf("%s -> %s", t1, t2), - Notify: false, - } - graph.AddEdge(t1, t2, edge) - testCases = append(testCases, test{ - name: "two resources and send/recv edge", - code: ` - test "t1" { - int64ptr => 42, - } - test "t2" { - int64ptr => 13, - } - - Test["t1"].hello -> Test["t2"].stringptr # send/recv - `, - graph: graph, - }) - } - { - graph, _ := pgraph.NewGraph("g") - t1, _ := engine.NewNamedResource("test", "t1") - x := t1.(*resources.TestRes) - stringptr := "this is meta" - x.StringPtr = &stringptr - m := &engine.MetaParams{ - Noop: true, // overwritten - Retry: -1, - Delay: 0, - Poll: 5, - Limit: 4.2, - Burst: 3, - Sema: []string{"foo:1", "bar:3"}, - Rewatch: false, - Realize: true, - } - x.SetMetaParams(m) - graph.AddVertex(t1) - testCases = append(testCases, test{ - name: "resource with meta params", - code: ` - test "t1" { - stringptr => "this is meta", - - Meta => struct{ - noop => false, - retry => -1, - delay => 0, - poll => 5, - limit => 4.2, - burst => 3, - sema => ["foo:1", "bar:3",], - rewatch => false, - realize => true, - reverse => true, - autoedge => true, - autogroup => true, - }, - Meta:noop => true, - Meta:reverse => true, - Meta:autoedge => true, - Meta:autogroup => true, - } - `, - graph: graph, - }) - } - - names := []string{} - for index, tc := range testCases { // run all the tests - name, code, fail, exp := tc.name, tc.code, tc.fail, tc.graph - - if name == "" { - name = "" - } - if util.StrInList(name, names) { - t.Errorf("test #%d: duplicate sub test name of: %s", index, name) - continue - } - names = append(names, name) - - //if index != 3 { // hack to run a subset (useful for debugging) - //if tc.name != "nil" { - // continue - //} - - t.Logf("\n\ntest #%d (%s) ----------------\n\n", index, name) - - str := strings.NewReader(code) - xast, err := parser.LexParse(str) - if err != nil { - t.Errorf("test #%d: lex/parse failed with: %+v", index, err) - continue - } - t.Logf("test #%d: AST: %+v", index, xast) - - data := &interfaces.Data{ - // TODO: add missing fields here if/when needed - Debug: testing.Verbose(), // set via the -test.v flag to `go test` - Logf: func(format string, v ...interface{}) { - t.Logf("ast: "+format, v...) - }, - } - // some of this might happen *after* interpolate in SetScope or Unify... - if err := xast.Init(data); err != nil { - t.Errorf("test #%d: FAIL", index) - t.Errorf("test #%d: could not init and validate AST: %+v", index, err) - return - } - - // these tests only work in certain cases, since this does not - // perform type unification, run the function graph engine, and - // only gives you limited results... don't expect normal code to - // run and produce meaningful things in this test... - graph, err := interpret.Interpret(xast) - - if !fail && err != nil { - t.Errorf("test #%d: interpret failed with: %+v", index, err) - continue - } - if fail && err == nil { - t.Errorf("test #%d: interpret passed, expected fail", index) - continue - } - - if fail { // can't process graph if it's nil - // TODO: match against expected error - t.Logf("test #%d: expected fail, error: %+v", index, err) - continue - } - - t.Logf("test #%d: graph: %+v", index, graph) - // TODO: improve: https://github.com/purpleidea/mgmt/issues/199 - if err := graph.GraphCmp(exp, vertexCmpFn, edgeCmpFn); err != nil { - t.Logf("test #%d: actual (g1): %v%s", index, graph, fullPrint(graph)) - t.Logf("test #%d: expected (g2): %v%s", index, exp, fullPrint(exp)) - t.Errorf("test #%d: cmp error:\n%v", index, err) - continue - } - - for i, v := range graph.Vertices() { - t.Logf("test #%d: vertex(%d): %+v", index, i, v) - } - for v1 := range graph.Adjacency() { - for v2, e := range graph.Adjacency()[v1] { - t.Logf("test #%d: edge(%+v): %+v -> %+v", index, e, v1, v2) - } - } - } -} diff --git a/lang/interpret_test/TestAstFunc1/changing-func.txtar b/lang/interpret_test/TestAstFunc1/changing-func.txtar index 0e4360c8c8..eb4d823818 100644 --- a/lang/interpret_test/TestAstFunc1/changing-func.txtar +++ b/lang/interpret_test/TestAstFunc1/changing-func.txtar @@ -21,41 +21,15 @@ $out2 = $fn2() test $out1 {} test $out2 {} -- OUTPUT -- -Edge: bool(false) -> call:funcgen(bool(false)) # b -Edge: bool(false) -> var(b) # var:b -Edge: bool(true) -> call:funcgen(bool(true)) # b -Edge: bool(true) -> var(b) # var:b -Edge: call:fn1() -> var(out1) # var:out1 -Edge: call:fn2() -> var(out2) # var:out2 -Edge: call:funcgen(bool(false)) -> call:fn2() # call:fn2 -Edge: call:funcgen(bool(true)) -> call:fn1() # call:fn1 -Edge: func() { str("hello") } -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # a -Edge: func() { str("hello") } -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # a -Edge: func() { str("world") } -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # b -Edge: func() { str("world") } -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # b -Edge: func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } -> call:funcgen(bool(false)) # call:funcgen -Edge: func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } -> call:funcgen(bool(true)) # call:funcgen -Edge: if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } -> func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } # body -Edge: if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } -> func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } # body -Edge: str("hello") -> func() { str("hello") } # body -Edge: str("world") -> func() { str("world") } # body -Edge: var(b) -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # c -Edge: var(b) -> if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } # c -Vertex: bool(false) -Vertex: bool(true) -Vertex: call:fn1() -Vertex: call:fn2() -Vertex: call:funcgen(bool(false)) -Vertex: call:funcgen(bool(true)) -Vertex: func() { str("hello") } -Vertex: func() { str("world") } -Vertex: func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } -Vertex: func(b) { if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } } -Vertex: if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } -Vertex: if( var(b) ) { func() { str("hello") } } else { func() { str("world") } } -Vertex: str("hello") -Vertex: str("world") -Vertex: var(b) -Vertex: var(b) -Vertex: var(out1) -Vertex: var(out2) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: call -> call # fn +Edge: call -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/classes-polyfuncs.txtar b/lang/interpret_test/TestAstFunc1/classes-polyfuncs.txtar index 7143bab217..7a571096e3 100644 --- a/lang/interpret_test/TestAstFunc1/classes-polyfuncs.txtar +++ b/lang/interpret_test/TestAstFunc1/classes-polyfuncs.txtar @@ -6,20 +6,19 @@ class c1($b) { test fmt.printf("len is: %d", len($b)) {} # len is 4 } -- OUTPUT -- -Edge: call:len(var(b)) -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # a -Edge: int(-37) -> list(int(13), int(42), int(0), int(-37)) # 3 -Edge: int(0) -> list(int(13), int(42), int(0), int(-37)) # 2 -Edge: int(13) -> list(int(13), int(42), int(0), int(-37)) # 0 -Edge: int(42) -> list(int(13), int(42), int(0), int(-37)) # 1 -Edge: list(int(13), int(42), int(0), int(-37)) -> var(b) # var:b -Edge: str("len is: %d") -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # format -Edge: var(b) -> call:len(var(b)) # 0 -Vertex: call:fmt.printf(str("len is: %d"), call:len(var(b))) -Vertex: call:len(var(b)) -Vertex: int(-37) -Vertex: int(0) -Vertex: int(13) -Vertex: int(42) -Vertex: list(int(13), int(42), int(0), int(-37)) -Vertex: str("len is: %d") -Vertex: var(b) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: const -> composite # 0 +Edge: const -> composite # 1 +Edge: const -> composite # 2 +Edge: const -> composite # 3 +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: composite +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/doubleclass.txtar b/lang/interpret_test/TestAstFunc1/doubleclass.txtar index 339c27a231..96b7398ee3 100644 --- a/lang/interpret_test/TestAstFunc1/doubleclass.txtar +++ b/lang/interpret_test/TestAstFunc1/doubleclass.txtar @@ -15,72 +15,51 @@ include foo(1) include foo(2) include foo(3) -- OUTPUT -- -Edge: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -> var(inside) # var:inside -Edge: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -> var(inside) # var:inside -Edge: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -> var(inside) # var:inside -Edge: call:_operator(str("+"), var(some_value1), var(some_value2)) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # a -Edge: call:_operator(str("+"), var(some_value1), var(some_value2)) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # a -Edge: call:_operator(str("+"), var(some_value1), var(some_value2)) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # a -Edge: int(1) -> var(num) # var:num -Edge: int(13) -> var(some_value2) # var:some_value2 -Edge: int(13) -> var(some_value2) # var:some_value2 -Edge: int(13) -> var(some_value2) # var:some_value2 -Edge: int(2) -> var(num) # var:num -Edge: int(3) -> var(num) # var:num -Edge: int(4) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # b -Edge: int(4) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # b -Edge: int(4) -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # b -Edge: int(42) -> var(some_value1) # var:some_value1 -Edge: int(42) -> var(some_value1) # var:some_value1 -Edge: int(42) -> var(some_value1) # var:some_value1 -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # op -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # op -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) # op -Edge: str("+") -> call:_operator(str("+"), var(some_value1), var(some_value2)) # op -Edge: str("+") -> call:_operator(str("+"), var(some_value1), var(some_value2)) # op -Edge: str("+") -> call:_operator(str("+"), var(some_value1), var(some_value2)) # op -Edge: str("test-%d-%d") -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # format -Edge: str("test-%d-%d") -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # format -Edge: str("test-%d-%d") -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # format -Edge: var(inside) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # b -Edge: var(inside) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # b -Edge: var(inside) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # b -Edge: var(num) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # a -Edge: var(num) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # a -Edge: var(num) -> call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) # a -Edge: var(some_value1) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # a -Edge: var(some_value1) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # a -Edge: var(some_value1) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # a -Edge: var(some_value2) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # b -Edge: var(some_value2) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # b -Edge: var(some_value2) -> call:_operator(str("+"), var(some_value1), var(some_value2)) # b -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(some_value1), var(some_value2)), int(4)) -Vertex: call:_operator(str("+"), var(some_value1), var(some_value2)) -Vertex: call:_operator(str("+"), var(some_value1), var(some_value2)) -Vertex: call:_operator(str("+"), var(some_value1), var(some_value2)) -Vertex: call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) -Vertex: call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) -Vertex: call:fmt.printf(str("test-%d-%d"), var(num), var(inside)) -Vertex: int(1) -Vertex: int(13) -Vertex: int(2) -Vertex: int(3) -Vertex: int(4) -Vertex: int(42) -Vertex: str("+") -Vertex: str("+") -Vertex: str("test-%d-%d") -Vertex: var(inside) -Vertex: var(inside) -Vertex: var(inside) -Vertex: var(num) -Vertex: var(num) -Vertex: var(num) -Vertex: var(some_value1) -Vertex: var(some_value1) -Vertex: var(some_value1) -Vertex: var(some_value2) -Vertex: var(some_value2) -Vertex: var(some_value2) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/doubleinclude.txtar b/lang/interpret_test/TestAstFunc1/doubleinclude.txtar index 7585f481dd..0be2e1b417 100644 --- a/lang/interpret_test/TestAstFunc1/doubleinclude.txtar +++ b/lang/interpret_test/TestAstFunc1/doubleinclude.txtar @@ -8,14 +8,7 @@ class c1($a) { } $foo = "hey" -- OUTPUT -- -Edge: str("hey") -> var(foo) # var:foo -Edge: str("hey") -> var(foo) # var:foo -Edge: str("t1") -> var(a) # var:a -Edge: str("t2") -> var(a) # var:a -Vertex: str("hey") -Vertex: str("t1") -Vertex: str("t2") -Vertex: var(a) -Vertex: var(a) -Vertex: var(foo) -Vertex: var(foo) +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/duplicate_resource.txtar b/lang/interpret_test/TestAstFunc1/duplicate_resource.txtar index ec9988dced..5a6fc4025f 100644 --- a/lang/interpret_test/TestAstFunc1/duplicate_resource.txtar +++ b/lang/interpret_test/TestAstFunc1/duplicate_resource.txtar @@ -17,13 +17,14 @@ pkg "cowsay" { state => "newest", } -- OUTPUT -- -Edge: str("hello world") -> call:fmt.printf(str("hello world")) # format -Vertex: call:fmt.printf(str("hello world")) -Vertex: str("/tmp/foo") -Vertex: str("/tmp/foo") -Vertex: str("cowsay") -Vertex: str("cowsay") -Vertex: str("hello world") -Vertex: str("hello world") -Vertex: str("installed") -Vertex: str("newest") +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/efficient-lambda.txtar b/lang/interpret_test/TestAstFunc1/efficient-lambda.txtar index 63ced61af2..e6249fc90b 100644 --- a/lang/interpret_test/TestAstFunc1/efficient-lambda.txtar +++ b/lang/interpret_test/TestAstFunc1/efficient-lambda.txtar @@ -10,33 +10,11 @@ $out2 = $prefixer("b") test $out1 {} # helloa test $out2 {} # hellob -- OUTPUT -- -Edge: call:_operator(str("+"), str("hello"), var(x)) -> func(x) { call:_operator(str("+"), str("hello"), var(x)) } # body -Edge: call:_operator(str("+"), str("hello"), var(x)) -> func(x) { call:_operator(str("+"), str("hello"), var(x)) } # body -Edge: call:prefixer(str("a")) -> var(out1) # var:out1 -Edge: call:prefixer(str("b")) -> var(out2) # var:out2 -Edge: func(x) { call:_operator(str("+"), str("hello"), var(x)) } -> call:prefixer(str("a")) # call:prefixer -Edge: func(x) { call:_operator(str("+"), str("hello"), var(x)) } -> call:prefixer(str("b")) # call:prefixer -Edge: str("+") -> call:_operator(str("+"), str("hello"), var(x)) # op -Edge: str("+") -> call:_operator(str("+"), str("hello"), var(x)) # op -Edge: str("a") -> call:prefixer(str("a")) # x -Edge: str("a") -> var(x) # var:x -Edge: str("b") -> call:prefixer(str("b")) # x -Edge: str("b") -> var(x) # var:x -Edge: str("hello") -> call:_operator(str("+"), str("hello"), var(x)) # a -Edge: str("hello") -> call:_operator(str("+"), str("hello"), var(x)) # a -Edge: var(x) -> call:_operator(str("+"), str("hello"), var(x)) # b -Edge: var(x) -> call:_operator(str("+"), str("hello"), var(x)) # b -Vertex: call:_operator(str("+"), str("hello"), var(x)) -Vertex: call:_operator(str("+"), str("hello"), var(x)) -Vertex: call:prefixer(str("a")) -Vertex: call:prefixer(str("b")) -Vertex: func(x) { call:_operator(str("+"), str("hello"), var(x)) } -Vertex: func(x) { call:_operator(str("+"), str("hello"), var(x)) } -Vertex: str("+") -Vertex: str("a") -Vertex: str("b") -Vertex: str("hello") -Vertex: var(out1) -Vertex: var(out2) -Vertex: var(x) -Vertex: var(x) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/empty-res-list-1.txtar b/lang/interpret_test/TestAstFunc1/empty-res-list-1.txtar index b8ddff7099..eb2cbf44cd 100644 --- a/lang/interpret_test/TestAstFunc1/empty-res-list-1.txtar +++ b/lang/interpret_test/TestAstFunc1/empty-res-list-1.txtar @@ -9,14 +9,12 @@ test $names {} # multiples resources, defined by list test ["hello", "world",] {} -- OUTPUT -- -Edge: list(str("hey")) -> var(names) # var:names -Edge: str("hello") -> list(str("hello"), str("world")) # 0 -Edge: str("hey") -> list(str("hey")) # 0 -Edge: str("world") -> list(str("hello"), str("world")) # 1 -Vertex: list(str("hello"), str("world")) -Vertex: list(str("hey")) -Vertex: str("hello") -Vertex: str("hey") -Vertex: str("name") -Vertex: str("world") -Vertex: var(names) +Edge: const -> composite # 0 +Edge: const -> composite # 0 +Edge: const -> composite # 1 +Vertex: composite +Vertex: composite +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph0.txtar b/lang/interpret_test/TestAstFunc1/graph0.txtar new file mode 100644 index 0000000000..54ed60980c --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph0.txtar @@ -0,0 +1,2 @@ +-- main.mcl -- +-- OUTPUT -- diff --git a/lang/interpret_test/TestAstFunc1/graph1.txtar b/lang/interpret_test/TestAstFunc1/graph1.txtar new file mode 100644 index 0000000000..01bfd8e5ea --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph1.txtar @@ -0,0 +1,5 @@ +-- main.mcl -- +# TODO: this should fail with an unused variable error! +$x = 42 +$y = $x +-- OUTPUT -- diff --git a/lang/interpret_test/TestAstFunc1/graph10.txtar b/lang/interpret_test/TestAstFunc1/graph10.txtar new file mode 100644 index 0000000000..029cbbc1f5 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph10.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +# this should be okay, because var is shadowed +$x = "hello" +if true { + $x = "world" # shadowed +} +test "t" { + stringptr => $x, +} +-- OUTPUT -- +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph11.txtar b/lang/interpret_test/TestAstFunc1/graph11.txtar new file mode 100644 index 0000000000..8af1f61630 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph11.txtar @@ -0,0 +1,14 @@ +-- main.mcl -- +# variable shadowing inner +# this should be okay, because var is shadowed +$x = "hello" +if true { + $x = "world" # shadowed + test "t" { + stringptr => $x, + } +} +-- OUTPUT -- +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph12.txtar b/lang/interpret_test/TestAstFunc1/graph12.txtar new file mode 100644 index 0000000000..4d908687c7 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph12.txtar @@ -0,0 +1,19 @@ +-- main.mcl -- +# variable shadowing both +# this should be okay, because var is shadowed +$x = "hello" +if true { + $x = "world" # shadowed + test "t2" { + stringptr => $x, + } +} +test "t1" { + stringptr => $x, +} +-- OUTPUT -- +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph13.txtar b/lang/interpret_test/TestAstFunc1/graph13.txtar new file mode 100644 index 0000000000..6bb23cad27 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph13.txtar @@ -0,0 +1,14 @@ +-- main.mcl -- +# duplicate resource +# these two are allowed because they are compatible +pkg "cowsay" { + state => "installed", +} +pkg "cowsay" { + state => "newest", +} +-- OUTPUT -- +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph14.txtar b/lang/interpret_test/TestAstFunc1/graph14.txtar new file mode 100644 index 0000000000..270188fdc2 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph14.txtar @@ -0,0 +1,7 @@ +-- main.mcl -- +# variable re-declaration and type change error +# this should fail b/c of variable re-declaration +$x = "wow" +$x = 99 # woops, but also a change of type :P +-- OUTPUT -- +# err: errSetScope: var `x` already exists in this scope diff --git a/lang/interpret_test/TestAstFunc1/graph2.txtar b/lang/interpret_test/TestAstFunc1/graph2.txtar new file mode 100644 index 0000000000..b5b83af291 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph2.txtar @@ -0,0 +1,6 @@ +-- main.mcl -- +# self-referential vars +$x = $y +$y = $x +-- OUTPUT -- +# err: errSetScope: recursive reference while setting scope: not a dag diff --git a/lang/interpret_test/TestAstFunc1/graph3.txtar b/lang/interpret_test/TestAstFunc1/graph3.txtar new file mode 100644 index 0000000000..7f7cd0a52a --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph3.txtar @@ -0,0 +1,11 @@ +-- main.mcl -- +# chained vars +test "t" { + int64ptr => $c, +} +$c = $b +$b = $a +$a = 42 +-- OUTPUT -- +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph4.txtar b/lang/interpret_test/TestAstFunc1/graph4.txtar new file mode 100644 index 0000000000..532467e745 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph4.txtar @@ -0,0 +1,7 @@ +-- main.mcl -- +# simple bool +if $b { +} +$b = true +-- OUTPUT -- +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph5.txtar b/lang/interpret_test/TestAstFunc1/graph5.txtar new file mode 100644 index 0000000000..831e09bd29 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph5.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +# simple operator +test "t" { + int64ptr => 42 + 13, +} +-- OUTPUT -- +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph6.txtar b/lang/interpret_test/TestAstFunc1/graph6.txtar new file mode 100644 index 0000000000..2fcc17ba27 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph6.txtar @@ -0,0 +1,18 @@ +-- main.mcl -- +# simple operators +test "t" { + int64ptr => 42 + 13 - 99, +} +-- OUTPUT -- +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph7.txtar b/lang/interpret_test/TestAstFunc1/graph7.txtar new file mode 100644 index 0000000000..f98331f59f --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph7.txtar @@ -0,0 +1,18 @@ +-- main.mcl -- +# nested resource and scoped var +if true { + test "t" { + int64ptr => $x, + } + $x = 42 + $i +} +$i = 13 +-- OUTPUT -- +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/graph8.txtar b/lang/interpret_test/TestAstFunc1/graph8.txtar new file mode 100644 index 0000000000..e0766e85f3 --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph8.txtar @@ -0,0 +1,9 @@ +-- main.mcl -- +# should be out of scope, and a compile error! +if $b { +} +if true { + $b = true +} +-- OUTPUT -- +# err: errSetScope: variable b not in scope diff --git a/lang/interpret_test/TestAstFunc1/graph9.txtar b/lang/interpret_test/TestAstFunc1/graph9.txtar new file mode 100644 index 0000000000..3fb117aa5b --- /dev/null +++ b/lang/interpret_test/TestAstFunc1/graph9.txtar @@ -0,0 +1,6 @@ +-- main.mcl -- +# this should fail b/c of variable re-declaration +$x = "hello" +$x = "world" # woops +-- OUTPUT -- +# err: errSetScope: var `x` already exists in this scope diff --git a/lang/interpret_test/TestAstFunc1/hello0.txtar b/lang/interpret_test/TestAstFunc1/hello0.txtar index cf353196b7..0129a4fc83 100644 --- a/lang/interpret_test/TestAstFunc1/hello0.txtar +++ b/lang/interpret_test/TestAstFunc1/hello0.txtar @@ -7,11 +7,9 @@ test "greeting" { anotherstr => fmt.printf("hello: %s", $s), } -- OUTPUT -- -Edge: str("hello: %s") -> call:fmt.printf(str("hello: %s"), var(s)) # format -Edge: str("world") -> var(s) # var:s -Edge: var(s) -> call:fmt.printf(str("hello: %s"), var(s)) # a -Vertex: call:fmt.printf(str("hello: %s"), var(s)) -Vertex: str("greeting") -Vertex: str("hello: %s") -Vertex: str("world") -Vertex: var(s) +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/importscope0.txtar b/lang/interpret_test/TestAstFunc1/importscope0.txtar index f3edfbc306..c057e5fcbc 100644 --- a/lang/interpret_test/TestAstFunc1/importscope0.txtar +++ b/lang/interpret_test/TestAstFunc1/importscope0.txtar @@ -16,13 +16,13 @@ class xclass { } } -- OUTPUT -- -Edge: call:os.is_debian() -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # c -Edge: if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } -> var(aaa) # var:aaa -Edge: str("bbb") -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # a -Edge: str("ccc") -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # b -Vertex: call:os.is_debian() -Vertex: if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } -Vertex: str("bbb") -Vertex: str("ccc") -Vertex: str("hello") -Vertex: var(aaa) +Edge: FuncValue -> call # fn +Edge: call -> if # c +Edge: const -> if # a +Edge: const -> if # b +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: if diff --git a/lang/interpret_test/TestAstFunc1/importscope1.txtar b/lang/interpret_test/TestAstFunc1/importscope1.txtar index 99fbd28591..c57de89001 100644 --- a/lang/interpret_test/TestAstFunc1/importscope1.txtar +++ b/lang/interpret_test/TestAstFunc1/importscope1.txtar @@ -16,4 +16,4 @@ class xclass { } } -- OUTPUT -- -# err: errSetScope: import scope `second.mcl` failed: local import of `second.mcl` failed: could not set scope from import: func `os.is_debian` does not exist in this scope +# err: errSetScope: func `os.is_debian` does not exist in this scope diff --git a/lang/interpret_test/TestAstFunc1/importscope2.txtar b/lang/interpret_test/TestAstFunc1/importscope2.txtar index 9985611eb6..bf9b8bdf7b 100644 --- a/lang/interpret_test/TestAstFunc1/importscope2.txtar +++ b/lang/interpret_test/TestAstFunc1/importscope2.txtar @@ -15,13 +15,13 @@ class xclass { } } -- OUTPUT -- -Edge: call:os.is_debian() -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # c -Edge: if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } -> var(aaa) # var:aaa -Edge: str("bbb") -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # a -Edge: str("ccc") -> if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } # b -Vertex: call:os.is_debian() -Vertex: if( call:os.is_debian() ) { str("bbb") } else { str("ccc") } -Vertex: str("bbb") -Vertex: str("ccc") -Vertex: str("hello") -Vertex: var(aaa) +Edge: FuncValue -> call # fn +Edge: call -> if # c +Edge: const -> if # a +Edge: const -> if # b +Vertex: FuncValue +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: if diff --git a/lang/interpret_test/TestAstFunc1/lambda-chained.txtar b/lang/interpret_test/TestAstFunc1/lambda-chained.txtar index df91e05d42..e4b299d25a 100644 --- a/lang/interpret_test/TestAstFunc1/lambda-chained.txtar +++ b/lang/interpret_test/TestAstFunc1/lambda-chained.txtar @@ -12,48 +12,14 @@ $out2 = $prefixer($out1) test $out1 {} test $out2 {} -- OUTPUT -- -Edge: call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) -> func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } # body -Edge: call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) -> func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } # body -Edge: call:_operator(str("+"), var(prefix), str(":")) -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # a -Edge: call:_operator(str("+"), var(prefix), str(":")) -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # a -Edge: call:prefixer(str("world")) -> var(out1) # var:out1 -Edge: call:prefixer(str("world")) -> var(out1) # var:out1 -Edge: call:prefixer(var(out1)) -> var(out2) # var:out2 -Edge: func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } -> call:prefixer(str("world")) # call:prefixer -Edge: func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } -> call:prefixer(var(out1)) # call:prefixer -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # op -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # op -Edge: str("+") -> call:_operator(str("+"), var(prefix), str(":")) # op -Edge: str("+") -> call:_operator(str("+"), var(prefix), str(":")) # op -Edge: str(":") -> call:_operator(str("+"), var(prefix), str(":")) # b -Edge: str(":") -> call:_operator(str("+"), var(prefix), str(":")) # b -Edge: str("hello") -> var(prefix) # var:prefix -Edge: str("hello") -> var(prefix) # var:prefix -Edge: str("world") -> call:prefixer(str("world")) # x -Edge: str("world") -> var(x) # var:x -Edge: var(out1) -> call:prefixer(var(out1)) # x -Edge: var(out1) -> var(x) # var:x -Edge: var(prefix) -> call:_operator(str("+"), var(prefix), str(":")) # a -Edge: var(prefix) -> call:_operator(str("+"), var(prefix), str(":")) # a -Edge: var(x) -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # b -Edge: var(x) -> call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) # b -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) -Vertex: call:_operator(str("+"), var(prefix), str(":")) -Vertex: call:_operator(str("+"), var(prefix), str(":")) -Vertex: call:prefixer(str("world")) -Vertex: call:prefixer(var(out1)) -Vertex: func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } -Vertex: func(x) { call:_operator(str("+"), call:_operator(str("+"), var(prefix), str(":")), var(x)) } -Vertex: str("+") -Vertex: str("+") -Vertex: str(":") -Vertex: str("hello") -Vertex: str("world") -Vertex: var(out1) -Vertex: var(out1) -Vertex: var(out2) -Vertex: var(prefix) -Vertex: var(prefix) -Vertex: var(x) -Vertex: var(x) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/polydoubleinclude.txtar b/lang/interpret_test/TestAstFunc1/polydoubleinclude.txtar index 364141b647..2eaa7ee418 100644 --- a/lang/interpret_test/TestAstFunc1/polydoubleinclude.txtar +++ b/lang/interpret_test/TestAstFunc1/polydoubleinclude.txtar @@ -10,34 +10,29 @@ class c1($a, $b) { } } -- OUTPUT -- -Edge: call:len(var(b)) -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # a -Edge: call:len(var(b)) -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # a -Edge: int(-37) -> list(int(13), int(42), int(0), int(-37)) # 3 -Edge: int(0) -> list(int(13), int(42), int(0), int(-37)) # 2 -Edge: int(13) -> list(int(13), int(42), int(0), int(-37)) # 0 -Edge: int(42) -> list(int(13), int(42), int(0), int(-37)) # 1 -Edge: list(int(13), int(42), int(0), int(-37)) -> var(b) # var:b -Edge: str("hello") -> var(b) # var:b -Edge: str("len is: %d") -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # format -Edge: str("len is: %d") -> call:fmt.printf(str("len is: %d"), call:len(var(b))) # format -Edge: str("t1") -> var(a) # var:a -Edge: str("t2") -> var(a) # var:a -Edge: var(b) -> call:len(var(b)) # 0 -Edge: var(b) -> call:len(var(b)) # 0 -Vertex: call:fmt.printf(str("len is: %d"), call:len(var(b))) -Vertex: call:fmt.printf(str("len is: %d"), call:len(var(b))) -Vertex: call:len(var(b)) -Vertex: call:len(var(b)) -Vertex: int(-37) -Vertex: int(0) -Vertex: int(13) -Vertex: int(42) -Vertex: list(int(13), int(42), int(0), int(-37)) -Vertex: str("hello") -Vertex: str("len is: %d") -Vertex: str("t1") -Vertex: str("t2") -Vertex: var(a) -Vertex: var(a) -Vertex: var(b) -Vertex: var(b) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: const -> composite # 0 +Edge: const -> composite # 1 +Edge: const -> composite # 2 +Edge: const -> composite # 3 +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: composite +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/resdupefields5.txtar b/lang/interpret_test/TestAstFunc1/resdupefields5.txtar index 16d6510657..acbbbbc911 100644 --- a/lang/interpret_test/TestAstFunc1/resdupefields5.txtar +++ b/lang/interpret_test/TestAstFunc1/resdupefields5.txtar @@ -1,24 +1,62 @@ -- main.mcl -- -# XXX: should error at graph unification, but we have a type unification bug -#test "test" { -# anotherstr => "test", -# -# Meta => true ?: struct{ -# noop => false, -# retry => -1, -# delay => 0, -# poll => 5, -# limit => 4.2, -# burst => 3, -# sema => ["foo:1", "bar:3",], -# rewatch => false, -# realize => true, -# reverse => true, -# autoedge => true, -# autogroup => true, -# }, -# Meta => true ?: struct{ -# noop => false, -# }, -#} +test "test" { + anotherstr => "test", + + Meta => true ?: struct{ + noop => false, + retry => -1, + retryreset => false, + delay => 0, + poll => 5, + limit => 4.2, + burst => 3, + reset => false, + sema => ["foo:1", "bar:3",], + rewatch => false, + realize => true, + reverse => true, + autoedge => true, + autogroup => true, + }, + # XXX: should error at graph unification, but we have a type unification bug + #Meta => true ?: struct{ + # noop => false, + #}, +} -- OUTPUT -- +Edge: composite -> composite # sema +Edge: const -> composite # 0 +Edge: const -> composite # 1 +Edge: const -> composite # autoedge +Edge: const -> composite # autogroup +Edge: const -> composite # burst +Edge: const -> composite # delay +Edge: const -> composite # limit +Edge: const -> composite # noop +Edge: const -> composite # poll +Edge: const -> composite # realize +Edge: const -> composite # reset +Edge: const -> composite # retry +Edge: const -> composite # retryreset +Edge: const -> composite # reverse +Edge: const -> composite # rewatch +Vertex: composite +Vertex: composite +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/returned-func.txtar b/lang/interpret_test/TestAstFunc1/returned-func.txtar index 305c45dcd0..146786da33 100644 --- a/lang/interpret_test/TestAstFunc1/returned-func.txtar +++ b/lang/interpret_test/TestAstFunc1/returned-func.txtar @@ -11,14 +11,8 @@ $out = $fn() test $out {} -- OUTPUT -- -Edge: call:fn() -> var(out) # var:out -Edge: call:funcgen() -> call:fn() # call:fn -Edge: func() { func() { str("hello") } } -> call:funcgen() # call:funcgen -Edge: func() { str("hello") } -> func() { func() { str("hello") } } # body -Edge: str("hello") -> func() { str("hello") } # body -Vertex: call:fn() -Vertex: call:funcgen() -Vertex: func() { func() { str("hello") } } -Vertex: func() { str("hello") } -Vertex: str("hello") -Vertex: var(out) +Edge: FuncValue -> call # fn +Edge: call -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: call diff --git a/lang/interpret_test/TestAstFunc1/returned-lambda.txtar b/lang/interpret_test/TestAstFunc1/returned-lambda.txtar index a5654107a6..93b17fb60b 100644 --- a/lang/interpret_test/TestAstFunc1/returned-lambda.txtar +++ b/lang/interpret_test/TestAstFunc1/returned-lambda.txtar @@ -10,14 +10,8 @@ $out = $fn() test $out {} -- OUTPUT -- -Edge: call:fn() -> var(out) # var:out -Edge: call:funcgen() -> call:fn() # call:fn -Edge: func() { func() { str("hello") } } -> call:funcgen() # call:funcgen -Edge: func() { str("hello") } -> func() { func() { str("hello") } } # body -Edge: str("hello") -> func() { str("hello") } # body -Vertex: call:fn() -Vertex: call:funcgen() -Vertex: func() { func() { str("hello") } } -Vertex: func() { str("hello") } -Vertex: str("hello") -Vertex: var(out) +Edge: FuncValue -> call # fn +Edge: call -> call # fn +Vertex: FuncValue +Vertex: call +Vertex: call diff --git a/lang/interpret_test/TestAstFunc1/shadowing1.txtar b/lang/interpret_test/TestAstFunc1/shadowing1.txtar index 7ee95026f3..22fe868b14 100644 --- a/lang/interpret_test/TestAstFunc1/shadowing1.txtar +++ b/lang/interpret_test/TestAstFunc1/shadowing1.txtar @@ -6,7 +6,5 @@ if true { } test $x {} -- OUTPUT -- -Edge: str("hello") -> var(x) # var:x -Vertex: bool(true) -Vertex: str("hello") -Vertex: var(x) +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/shadowing2.txtar b/lang/interpret_test/TestAstFunc1/shadowing2.txtar index 34f5dd9c01..3b297a6fe5 100644 --- a/lang/interpret_test/TestAstFunc1/shadowing2.txtar +++ b/lang/interpret_test/TestAstFunc1/shadowing2.txtar @@ -6,7 +6,5 @@ if true { test $x {} } -- OUTPUT -- -Edge: str("world") -> var(x) # var:x -Vertex: bool(true) -Vertex: str("world") -Vertex: var(x) +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/simple-func1.txtar b/lang/interpret_test/TestAstFunc1/simple-func1.txtar index 5cc59bc7fb..72a0cee0e3 100644 --- a/lang/interpret_test/TestAstFunc1/simple-func1.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-func1.txtar @@ -7,10 +7,6 @@ $out1 = answer() test $out1 {} -- OUTPUT -- -Edge: call:answer() -> var(out1) # var:out1 -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: str("the answer is 42") -> func() { str("the answer is 42") } # body -Vertex: call:answer() -Vertex: func() { str("the answer is 42") } -Vertex: str("the answer is 42") -Vertex: var(out1) +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call diff --git a/lang/interpret_test/TestAstFunc1/simple-func2.txtar b/lang/interpret_test/TestAstFunc1/simple-func2.txtar index d0c1976ca2..e697c85774 100644 --- a/lang/interpret_test/TestAstFunc1/simple-func2.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-func2.txtar @@ -8,19 +8,13 @@ $out2 = answer() test $out1 + $out2 {} -- OUTPUT -- -Edge: call:answer() -> var(out1) # var:out1 -Edge: call:answer() -> var(out2) # var:out2 -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: str("+") -> call:_operator(str("+"), var(out1), var(out2)) # op -Edge: str("the answer is 42") -> func() { str("the answer is 42") } # body -Edge: var(out1) -> call:_operator(str("+"), var(out1), var(out2)) # a -Edge: var(out2) -> call:_operator(str("+"), var(out1), var(out2)) # b -Vertex: call:_operator(str("+"), var(out1), var(out2)) -Vertex: call:answer() -Vertex: call:answer() -Vertex: func() { str("the answer is 42") } -Vertex: str("+") -Vertex: str("the answer is 42") -Vertex: var(out1) -Vertex: var(out2) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar b/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar index 4317631666..965be03880 100644 --- a/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-lambda1.txtar @@ -10,10 +10,6 @@ $out = $answer() test $out {} -- OUTPUT -- -Edge: call:answer() -> var(out) # var:out -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: str("the answer is 42") -> func() { str("the answer is 42") } # body -Vertex: call:answer() -Vertex: func() { str("the answer is 42") } -Vertex: str("the answer is 42") -Vertex: var(out) +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: call diff --git a/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar b/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar index b879b30f62..c5c3b0529d 100644 --- a/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar +++ b/lang/interpret_test/TestAstFunc1/simple-lambda2.txtar @@ -11,19 +11,13 @@ $out2 = $answer() test $out1 + $out2 {} -- OUTPUT -- -Edge: call:answer() -> var(out1) # var:out1 -Edge: call:answer() -> var(out2) # var:out2 -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: func() { str("the answer is 42") } -> call:answer() # call:answer -Edge: str("+") -> call:_operator(str("+"), var(out1), var(out2)) # op -Edge: str("the answer is 42") -> func() { str("the answer is 42") } # body -Edge: var(out1) -> call:_operator(str("+"), var(out1), var(out2)) # a -Edge: var(out2) -> call:_operator(str("+"), var(out1), var(out2)) # b -Vertex: call:_operator(str("+"), var(out1), var(out2)) -Vertex: call:answer() -Vertex: call:answer() -Vertex: func() { str("the answer is 42") } -Vertex: str("+") -Vertex: str("the answer is 42") -Vertex: var(out1) -Vertex: var(out2) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/slow_unification0.txtar b/lang/interpret_test/TestAstFunc1/slow_unification0.txtar index e5203254d1..adc5a4dbc4 100644 --- a/lang/interpret_test/TestAstFunc1/slow_unification0.txtar +++ b/lang/interpret_test/TestAstFunc1/slow_unification0.txtar @@ -52,91 +52,90 @@ if $state == "three" { Exec["timer"] -> Kv["${ns}"] } -- OUTPUT -- -Edge: call:_operator(str("=="), var(state), str("default")) -> call:_operator(str("||"), call:_operator(str("=="), var(state), str("one")), call:_operator(str("=="), var(state), str("default"))) # b -Edge: call:_operator(str("=="), var(state), str("one")) -> call:_operator(str("||"), call:_operator(str("=="), var(state), str("one")), call:_operator(str("=="), var(state), str("default"))) # a -Edge: call:maplookup(var(exchanged), var(hostname), str("default")) -> var(state) # var:state -Edge: call:maplookup(var(exchanged), var(hostname), str("default")) -> var(state) # var:state -Edge: call:maplookup(var(exchanged), var(hostname), str("default")) -> var(state) # var:state -Edge: call:maplookup(var(exchanged), var(hostname), str("default")) -> var(state) # var:state -Edge: call:world.kvlookup(var(ns)) -> var(exchanged) # var:exchanged -Edge: str("") -> var(hostname) # var:hostname -Edge: str("==") -> call:_operator(str("=="), var(state), str("default")) # op -Edge: str("==") -> call:_operator(str("=="), var(state), str("one")) # op -Edge: str("==") -> call:_operator(str("=="), var(state), str("three")) # op -Edge: str("==") -> call:_operator(str("=="), var(state), str("two")) # op -Edge: str("default") -> call:_operator(str("=="), var(state), str("default")) # b -Edge: str("default") -> call:maplookup(var(exchanged), var(hostname), str("default")) # default -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("estate") -> var(ns) # var:ns -Edge: str("one") -> call:_operator(str("=="), var(state), str("one")) # b -Edge: str("three") -> call:_operator(str("=="), var(state), str("three")) # b -Edge: str("two") -> call:_operator(str("=="), var(state), str("two")) # b -Edge: str("||") -> call:_operator(str("||"), call:_operator(str("=="), var(state), str("one")), call:_operator(str("=="), var(state), str("default"))) # op -Edge: var(exchanged) -> call:maplookup(var(exchanged), var(hostname), str("default")) # map -Edge: var(hostname) -> call:maplookup(var(exchanged), var(hostname), str("default")) # key -Edge: var(ns) -> call:world.kvlookup(var(ns)) # namespace -Edge: var(state) -> call:_operator(str("=="), var(state), str("default")) # a -Edge: var(state) -> call:_operator(str("=="), var(state), str("one")) # a -Edge: var(state) -> call:_operator(str("=="), var(state), str("three")) # a -Edge: var(state) -> call:_operator(str("=="), var(state), str("two")) # a -Vertex: call:_operator(str("=="), var(state), str("default")) -Vertex: call:_operator(str("=="), var(state), str("one")) -Vertex: call:_operator(str("=="), var(state), str("three")) -Vertex: call:_operator(str("=="), var(state), str("two")) -Vertex: call:_operator(str("||"), call:_operator(str("=="), var(state), str("one")), call:_operator(str("=="), var(state), str("default"))) -Vertex: call:maplookup(var(exchanged), var(hostname), str("default")) -Vertex: call:world.kvlookup(var(ns)) -Vertex: str("") -Vertex: str("/tmp/mgmt/state") -Vertex: str("/tmp/mgmt/state") -Vertex: str("/tmp/mgmt/state") -Vertex: str("/usr/bin/sleep 1s") -Vertex: str("/usr/bin/sleep 1s") -Vertex: str("/usr/bin/sleep 1s") -Vertex: str("==") -Vertex: str("==") -Vertex: str("==") -Vertex: str("==") -Vertex: str("default") -Vertex: str("default") -Vertex: str("estate") -Vertex: str("one") -Vertex: str("one") -Vertex: str("state: one\n") -Vertex: str("state: three\n") -Vertex: str("state: two\n") -Vertex: str("three") -Vertex: str("three") -Vertex: str("timer") -Vertex: str("timer") -Vertex: str("timer") -Vertex: str("timer") -Vertex: str("timer") -Vertex: str("timer") -Vertex: str("two") -Vertex: str("two") -Vertex: str("||") -Vertex: var(exchanged) -Vertex: var(hostname) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(ns) -Vertex: var(state) -Vertex: var(state) -Vertex: var(state) -Vertex: var(state) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/static-function0.txtar b/lang/interpret_test/TestAstFunc1/static-function0.txtar index 4adeaeea31..9e824c8eb3 100644 --- a/lang/interpret_test/TestAstFunc1/static-function0.txtar +++ b/lang/interpret_test/TestAstFunc1/static-function0.txtar @@ -16,15 +16,15 @@ test "greeting3" { anotherstr => $fn(), } -- OUTPUT -- -Edge: func() { str("hello world") } -> call:fn() # call:fn -Edge: func() { str("hello world") } -> call:fn() # call:fn -Edge: func() { str("hello world") } -> call:fn() # call:fn -Edge: str("hello world") -> func() { str("hello world") } # body -Vertex: call:fn() -Vertex: call:fn() -Vertex: call:fn() -Vertex: func() { str("hello world") } -Vertex: str("greeting1") -Vertex: str("greeting2") -Vertex: str("greeting3") -Vertex: str("hello world") +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc1/static-function1.txtar b/lang/interpret_test/TestAstFunc1/static-function1.txtar index a2b2dc4dbb..da23a96a76 100644 --- a/lang/interpret_test/TestAstFunc1/static-function1.txtar +++ b/lang/interpret_test/TestAstFunc1/static-function1.txtar @@ -18,59 +18,15 @@ test "greeting3" { anotherstr => $fn(), } -- OUTPUT -- -Edge: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -> func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } # body -Edge: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -> func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } # body -Edge: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -> func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } # body -Edge: call:_operator(str("+"), var(s1), str(" ")) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # a -Edge: call:_operator(str("+"), var(s1), str(" ")) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # a -Edge: call:_operator(str("+"), var(s1), str(" ")) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # a -Edge: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -> call:fn() # call:fn -Edge: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -> call:fn() # call:fn -Edge: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -> call:fn() # call:fn -Edge: str(" ") -> call:_operator(str("+"), var(s1), str(" ")) # b -Edge: str(" ") -> call:_operator(str("+"), var(s1), str(" ")) # b -Edge: str(" ") -> call:_operator(str("+"), var(s1), str(" ")) # b -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # op -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # op -Edge: str("+") -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # op -Edge: str("+") -> call:_operator(str("+"), var(s1), str(" ")) # op -Edge: str("+") -> call:_operator(str("+"), var(s1), str(" ")) # op -Edge: str("+") -> call:_operator(str("+"), var(s1), str(" ")) # op -Edge: str("hello") -> var(s1) # var:s1 -Edge: str("hello") -> var(s1) # var:s1 -Edge: str("hello") -> var(s1) # var:s1 -Edge: str("world") -> var(s2) # var:s2 -Edge: str("world") -> var(s2) # var:s2 -Edge: str("world") -> var(s2) # var:s2 -Edge: var(s1) -> call:_operator(str("+"), var(s1), str(" ")) # a -Edge: var(s1) -> call:_operator(str("+"), var(s1), str(" ")) # a -Edge: var(s1) -> call:_operator(str("+"), var(s1), str(" ")) # a -Edge: var(s2) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # b -Edge: var(s2) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # b -Edge: var(s2) -> call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) # b -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -Vertex: call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) -Vertex: call:_operator(str("+"), var(s1), str(" ")) -Vertex: call:_operator(str("+"), var(s1), str(" ")) -Vertex: call:_operator(str("+"), var(s1), str(" ")) -Vertex: call:fn() -Vertex: call:fn() -Vertex: call:fn() -Vertex: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -Vertex: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -Vertex: func() { call:_operator(str("+"), call:_operator(str("+"), var(s1), str(" ")), var(s2)) } -Vertex: str(" ") -Vertex: str("+") -Vertex: str("+") -Vertex: str("greeting1") -Vertex: str("greeting2") -Vertex: str("greeting3") -Vertex: str("hello") -Vertex: str("world") -Vertex: var(s1) -Vertex: var(s1) -Vertex: var(s1) -Vertex: var(s2) -Vertex: var(s2) -Vertex: var(s2) +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Edge: FuncValue -> call # fn +Vertex: FuncValue +Vertex: FuncValue +Vertex: FuncValue +Vertex: call +Vertex: call +Vertex: call +Vertex: const +Vertex: const +Vertex: const diff --git a/lang/interpret_test/TestAstFunc2/chained-vars.txtar b/lang/interpret_test/TestAstFunc2/chained-vars.txtar new file mode 100644 index 0000000000..72ffdcf299 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/chained-vars.txtar @@ -0,0 +1,10 @@ +-- main.mcl -- +import "fmt" + +$zero = 0 +$one = $zero + 1 +$two = $one + 1 # needs a chain to panic + +test fmt.printf("%d%d%d", $zero, $one, $two) {} +-- OUTPUT -- +Vertex: test[012] diff --git a/lang/interpret_test/TestAstFunc2/class-capture8.txtar b/lang/interpret_test/TestAstFunc2/class-capture8.txtar index 7d82d6b8eb..b19d726b81 100644 --- a/lang/interpret_test/TestAstFunc2/class-capture8.txtar +++ b/lang/interpret_test/TestAstFunc2/class-capture8.txtar @@ -1,5 +1,6 @@ -- main.mcl -- $x1 = "bad1" +$x2 = "also bad" include defs.foo import "defs.mcl" # out of order for fun diff --git a/lang/interpret_test/TestAstFunc2/complex-example.txtar b/lang/interpret_test/TestAstFunc2/complex-example.txtar new file mode 100644 index 0000000000..197e76125c --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/complex-example.txtar @@ -0,0 +1,25 @@ +-- main.mcl -- +import "fmt" +import "iter" + +# function expression +$id1 = func($x str) { # definition site + $x +} +$id2 = func($x str) { + $x + $x +} + +$generate = func($idn) { + $idn("foo") # 1 call site, 2 calls +} + +$foo = iter.map([$id1, $id2,], $generate) + +#test $foo[0] {} +#test $foo[1] {} +test listlookup($foo, 0, "fail") {} # TODO: add syntactic sugar for listlookup +test listlookup($foo, 1, "fail") {} # TODO: add syntactic sugar for listlookup +-- OUTPUT -- +Vertex: test[foo] +Vertex: test[foofoo] diff --git a/lang/interpret_test/TestAstFunc2/definition-site-scope.txtar b/lang/interpret_test/TestAstFunc2/definition-site-scope.txtar new file mode 100644 index 0000000000..f3d4979b94 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/definition-site-scope.txtar @@ -0,0 +1,19 @@ +-- main.mcl -- +$x = "top-level" +func f1() { + $x + "1" +} +$f2 = func() { + $x + "2" +} +$call_f1 = func($x) { + f1() + $x +} +$call_f2 = func($x) { + $f2() + $x +} +test $call_f1("!") {} +test $call_f2("?") {} +-- OUTPUT -- +Vertex: test[top-level1!] +Vertex: test[top-level2?] diff --git a/lang/interpret_test/TestAstFunc2/func-duplicate-arg.txtar b/lang/interpret_test/TestAstFunc2/func-duplicate-arg.txtar new file mode 100644 index 0000000000..da81f2d2fe --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/func-duplicate-arg.txtar @@ -0,0 +1,11 @@ +-- main.mcl -- +# function statement +func stradd($x) { + $x + $x +} + +$x1 = stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[heyhey] diff --git a/lang/interpret_test/TestAstFunc2/func-math1.txtar b/lang/interpret_test/TestAstFunc2/func-math1.txtar index 0c50de85bd..bf06d8b7c9 100644 --- a/lang/interpret_test/TestAstFunc2/func-math1.txtar +++ b/lang/interpret_test/TestAstFunc2/func-math1.txtar @@ -2,20 +2,14 @@ import "fmt" # function statement -func sq1($x, $y) { - $y + $x * $x +func sq1($x) { + $x + $x + #$x + 3 + #6 } -# function expression -$sq2 = func($x, $y) { - $y + $x * $x -} - -$x1 = sq1(3, 4) # 3^2 + 4 = 13 -$x2 = $sq2(7, -7) # 7^2 + 2 = 42 +$x1 = sq1(3) # 3^2 + 4 = 13 test fmt.printf("sq1: %d", $x1) {} -test fmt.printf("sq2: %d", $x2) {} -- OUTPUT -- -Vertex: test[sq1: 13] -Vertex: test[sq2: 42] +Vertex: test[sq1: 6] diff --git a/lang/interpret_test/TestAstFunc2/func-math2.txtar b/lang/interpret_test/TestAstFunc2/func-math2.txtar new file mode 100644 index 0000000000..7f797c8d6b --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/func-math2.txtar @@ -0,0 +1,17 @@ +-- main.mcl -- +import "fmt" +import "math" + +# XXX: do the input nodes to _operator look weird here? in my testing I saw +# fortytwo appear, and then disappear before we finished running... +# function statement +func sq1($x) { + $x + 3 +} + +$x1 = sq1(math.fortytwo()) + +test fmt.printf("sq1: %d", $x1) {} + +-- OUTPUT -- +Vertex: test[sq1: 45] diff --git a/lang/interpret_test/TestAstFunc2/func-one-arg.txtar b/lang/interpret_test/TestAstFunc2/func-one-arg.txtar new file mode 100644 index 0000000000..7f43aa63aa --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/func-one-arg.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" + +# function statement +func stradd($x) { + $x + "there" +} + +$x1 = stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[heythere] diff --git a/lang/interpret_test/TestAstFunc2/func-unused-arg.txtar b/lang/interpret_test/TestAstFunc2/func-unused-arg.txtar new file mode 100644 index 0000000000..912d574642 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/func-unused-arg.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" + +# function statement +func stradd($x) { + "nothing" +} + +$x1 = stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[nothing] diff --git a/lang/interpret_test/TestAstFunc2/identity-function-statement.txtar b/lang/interpret_test/TestAstFunc2/identity-function-statement.txtar new file mode 100644 index 0000000000..215088f6a9 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/identity-function-statement.txtar @@ -0,0 +1,9 @@ +-- main.mcl -- +# function statement +func identity($x) { + $x +} + +test identity("hey") {} +-- OUTPUT -- +Vertex: test[hey] diff --git a/lang/interpret_test/TestAstFunc2/if-0.txtar b/lang/interpret_test/TestAstFunc2/if-0.txtar new file mode 100644 index 0000000000..4a38952abe --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/if-0.txtar @@ -0,0 +1,10 @@ +-- main.mcl -- +$out1 = if true { + "hello" +} else { + "world" +} + +test $out1 {} +-- OUTPUT -- +Vertex: test[hello] diff --git a/lang/interpret_test/TestAstFunc2/if-in-func-0.txtar b/lang/interpret_test/TestAstFunc2/if-in-func-0.txtar new file mode 100644 index 0000000000..7d3c1dc0ac --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/if-in-func-0.txtar @@ -0,0 +1,15 @@ +-- main.mcl -- +# this can return changing functions, and could be optimized, too +$fn = func($b) { + if true { + "hello" + } else { + "world" + } +} + +$out1 = $fn(true) + +test $out1 {} +-- OUTPUT -- +Vertex: test[hello] diff --git a/lang/interpret_test/TestAstFunc2/invalid-function-call.txtar b/lang/interpret_test/TestAstFunc2/invalid-function-call.txtar index 92d1a3b289..4be4c38a4e 100644 --- a/lang/interpret_test/TestAstFunc2/invalid-function-call.txtar +++ b/lang/interpret_test/TestAstFunc2/invalid-function-call.txtar @@ -10,4 +10,4 @@ print "msg" { msg => fmt.printf("notfn: %d", $x), } -- OUTPUT -- -# err: errUnify: expected: Func, got: Int +# err: errUnify: func arg count differs diff --git a/lang/interpret_test/TestAstFunc2/lambda-duplicate-arg.txtar b/lang/interpret_test/TestAstFunc2/lambda-duplicate-arg.txtar new file mode 100644 index 0000000000..67ca8114d3 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/lambda-duplicate-arg.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" + +# function statement +$stradd = func($x) { + $x + $x +} + +$x1 = $stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[heyhey] diff --git a/lang/interpret_test/TestAstFunc2/lambda-no-args1.txtar b/lang/interpret_test/TestAstFunc2/lambda-no-args1.txtar new file mode 100644 index 0000000000..5b55d9bb1b --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/lambda-no-args1.txtar @@ -0,0 +1,8 @@ +-- main.mcl -- +$hey = func() { + "hello" +} + +test $hey() {} +-- OUTPUT -- +Vertex: test[hello] diff --git a/lang/interpret_test/TestAstFunc2/lambda-one-arg.txtar b/lang/interpret_test/TestAstFunc2/lambda-one-arg.txtar new file mode 100644 index 0000000000..bae4706cbe --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/lambda-one-arg.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" + +# function statement +$stradd = func($x) { + $x + "there" +} + +$x1 = $stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[heythere] diff --git a/lang/interpret_test/TestAstFunc2/lambda-one-func.txtar b/lang/interpret_test/TestAstFunc2/lambda-one-func.txtar new file mode 100644 index 0000000000..7421121a99 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/lambda-one-func.txtar @@ -0,0 +1,6 @@ +-- main.mcl -- +import "fmt" + +test fmt.printf("hello: %d", 42) {} +-- OUTPUT -- +Vertex: test[hello: 42] diff --git a/lang/interpret_test/TestAstFunc2/lambda-unused-arg.txtar b/lang/interpret_test/TestAstFunc2/lambda-unused-arg.txtar new file mode 100644 index 0000000000..9c1743ed3c --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/lambda-unused-arg.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" + +# function statement +$stradd = func($x) { + "nothing" +} + +$x1 = $stradd("hey") + +test $x1 {} +-- OUTPUT -- +Vertex: test[nothing] diff --git a/lang/interpret_test/TestAstFunc2/listlookup.txtar b/lang/interpret_test/TestAstFunc2/listlookup.txtar new file mode 100644 index 0000000000..0dc2371866 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/listlookup.txtar @@ -0,0 +1,20 @@ +-- main.mcl -- +import "fmt" +import "iter" + +$l1 = ["a", "b", "c",] + +$l2 = [$l1, ["hello", "world",],] + +#test $l1[0] {} +#test $l1[1] {} +test listlookup($l1, 0, "fail") {} # TODO: add syntactic sugar for listlookup +test listlookup($l1, 2, "fail") {} # TODO: add syntactic sugar for listlookup +test listlookup($l1, 3, "pass") {} # TODO: add syntactic sugar for listlookup +test listlookup($l2, 1, ["fail",]) {} # TODO: add syntactic sugar for listlookup +-- OUTPUT -- +Vertex: test[a] +Vertex: test[c] +Vertex: test[pass] +Vertex: test[hello] +Vertex: test[world] diff --git a/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar b/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar new file mode 100644 index 0000000000..d78a303449 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/map-ignore-arg.txtar @@ -0,0 +1,10 @@ +-- main.mcl -- +import "iter" + +$fn = func($x) { # ignore arg + "hey" +} + +test iter.map([1, 2, 3,], $fn) {} +-- OUTPUT -- +Vertex: test[hey] diff --git a/lang/interpret_test/TestAstFunc2/map-iterator0.txtar.TODO b/lang/interpret_test/TestAstFunc2/map-iterator0.txtar similarity index 100% rename from lang/interpret_test/TestAstFunc2/map-iterator0.txtar.TODO rename to lang/interpret_test/TestAstFunc2/map-iterator0.txtar diff --git a/lang/interpret_test/TestAstFunc2/map-iterator1.txtar.TODO b/lang/interpret_test/TestAstFunc2/map-iterator1.txtar similarity index 100% rename from lang/interpret_test/TestAstFunc2/map-iterator1.txtar.TODO rename to lang/interpret_test/TestAstFunc2/map-iterator1.txtar diff --git a/lang/interpret_test/TestAstFunc2/map-iterator2.txtar.TODO b/lang/interpret_test/TestAstFunc2/map-iterator2.txtar similarity index 100% rename from lang/interpret_test/TestAstFunc2/map-iterator2.txtar.TODO rename to lang/interpret_test/TestAstFunc2/map-iterator2.txtar diff --git a/lang/interpret_test/TestAstFunc2/map-iterator3.txtar.TODO b/lang/interpret_test/TestAstFunc2/map-iterator3.txtar similarity index 100% rename from lang/interpret_test/TestAstFunc2/map-iterator3.txtar.TODO rename to lang/interpret_test/TestAstFunc2/map-iterator3.txtar diff --git a/lang/interpret_test/TestAstFunc2/map-iterator4.txtar.TODO b/lang/interpret_test/TestAstFunc2/map-iterator4.txtar similarity index 100% rename from lang/interpret_test/TestAstFunc2/map-iterator4.txtar.TODO rename to lang/interpret_test/TestAstFunc2/map-iterator4.txtar diff --git a/lang/interpret_test/TestAstFunc2/map-numbered.txtar b/lang/interpret_test/TestAstFunc2/map-numbered.txtar new file mode 100644 index 0000000000..9edbb4b2f5 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/map-numbered.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +import "fmt" +import "iter" + +$fn = func($x) { + fmt.printf("hey%d", $x) +} + +test iter.map([1, 2, 3,], $fn) {} +-- OUTPUT -- +Vertex: test[hey1] +Vertex: test[hey2] +Vertex: test[hey3] diff --git a/lang/interpret_test/TestAstFunc2/res0.txtar b/lang/interpret_test/TestAstFunc2/res0.txtar new file mode 100644 index 0000000000..54ed60980c --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/res0.txtar @@ -0,0 +1,2 @@ +-- main.mcl -- +-- OUTPUT -- diff --git a/lang/interpret_test/TestAstFunc2/res1.txtar b/lang/interpret_test/TestAstFunc2/res1.txtar new file mode 100644 index 0000000000..6a2f0e615b --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/res1.txtar @@ -0,0 +1,6 @@ +-- main.mcl -- +test "t1" { + stringptr => 42, # int, not str +} +-- OUTPUT -- +# err: errUnify: can't unify, invariant illogicality with equals: base kind does not match (Int != Str) diff --git a/lang/interpret_test/TestAstFunc2/res2.txtar b/lang/interpret_test/TestAstFunc2/res2.txtar new file mode 100644 index 0000000000..a27e04a1db --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/res2.txtar @@ -0,0 +1,8 @@ +-- main.mcl -- +test "t1" { + int64ptr => 42, + stringptr => "okay cool", + int8ptrptrptr => 127, # super nested +} +-- OUTPUT -- +Vertex: test[t1] diff --git a/lang/interpret_test/TestAstFunc2/res3.txtar b/lang/interpret_test/TestAstFunc2/res3.txtar new file mode 100644 index 0000000000..d22ebb2e4e --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/res3.txtar @@ -0,0 +1,13 @@ +-- main.mcl -- +test "t1" { + int64ptr => 42, +} +test "t2" { + int64ptr => 13, +} + +Test["t1"].hello -> Test["t2"].stringptr # send/recv +-- OUTPUT -- +Edge: test[t1] -> test[t2] # test[t1] -> test[t2] +Vertex: test[t1] +Vertex: test[t2] diff --git a/lang/interpret_test/TestAstFunc2/res4.txtar b/lang/interpret_test/TestAstFunc2/res4.txtar new file mode 100644 index 0000000000..59d6757f15 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/res4.txtar @@ -0,0 +1,37 @@ +-- main.mcl -- +test "t1" { + stringptr => "this is meta", + + #Meta => struct{ + # noop => false, + # retry => -1, + # retryreset => false, + # delay => 0, + # poll => 5, + # limit => 4.2, + # burst => 3, + # reset => false, + # sema => ["foo:1", "bar:3",], + # rewatch => false, + # realize => true, + # reverse => true, + # autoedge => true, + # autogroup => true, + #}, + Meta:noop => false, + Meta:retry => -1, + Meta:retryreset => false, + Meta:delay => 0, + Meta:poll => 5, + Meta:limit => 4.2, + Meta:burst => 3, + Meta:reset => false, + Meta:sema => ["foo:1", "bar:3",], + Meta:rewatch => false, + Meta:realize => true, + Meta:reverse => true, + Meta:autoedge => true, + Meta:autogroup => true, +} +-- OUTPUT -- +Vertex: test[t1] diff --git a/lang/interpret_test/TestAstFunc2/template-1.txtar b/lang/interpret_test/TestAstFunc2/template-1.txtar new file mode 100644 index 0000000000..0c98aba36f --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/template-1.txtar @@ -0,0 +1,8 @@ +-- main.mcl -- +$v = 42 +$x = template("hello", $v) # redirect var for harder unification +test $x { + #anotherstr => $x, +} +-- OUTPUT -- +Vertex: test[hello] diff --git a/lang/interpret_test/TestAstFunc2/very-complex-example.txtar b/lang/interpret_test/TestAstFunc2/very-complex-example.txtar new file mode 100644 index 0000000000..94843f0ac3 --- /dev/null +++ b/lang/interpret_test/TestAstFunc2/very-complex-example.txtar @@ -0,0 +1,78 @@ +-- main.mcl -- +# This code should fail during type-checking. +# +# $id1 and $id2 are both polymorphic functions, which can be specialized to a +# different monomorphic type at every use site. In simple code, this +# specialization happens when the polymorphic function is called, but in more +# complex code, the specialization happens earlier, when the polymorphic +# function is passed as an argument to higher-order functions such as $generate +# and iter.map. +# +# In: +# +# $id1 = func($x str) { # definition site +# $x +# } +# $id2 = func($x str) { +# $x + $x +# } +# +# $generate = func($idn) { +# $idn("foo") # 1 call site, 2 calls +# } +# +# $foo = iter.map([$id1, $id2,], $generate) +# +# $generate is specialized to `func(func(str) str) str`, and $id1 and $id2 are +# specialized to `func(str) str`. +# +# In: +# +# $id1 = func($x) { # definition site +# $x +# } +# $id2 = func($x) { +# $x + $x +# } +# +# $generate = func($idn) { +# fmt.printf("%s %d", +# $idn("foo"), # 1 call site, 2 calls +# $idn(42) +# ) +# } +# +# $foo = iter.map([$id1, $id2,], $generate) +# +# $idn cannot be given a monomorphic type, since it is used both as a +# `func(str) str` and as a `func(int) int`. Therefore, $generate cannot be +# given a monomorphic type either, and neither can the call to iter.map. + +import "fmt" +import "iter" + +# function expression +$id1 = func($x) { # definition site + $x +} +$id2 = func($x) { + $x + $x +} + +#$str = $id1("foo") +#$int = $id1(42) + +$generate = func($idn) { + fmt.printf("%s %d", + $idn("foo"), # 1 call site, 2 calls + $idn(42) + ) +} + +# this code should be rejected during type unification +$foo = iter.map([$id1, $id2,], $generate) + +#test $foo[0] {} +test listlookup($foo, 0, "fail") {} # TODO: add syntactic sugar for listlookup +-- OUTPUT -- +# err: errUnify: can't unify, invariant illogicality with equals: base kind does not match (Str != Int) diff --git a/lang/lang.go b/lang/lang.go index e8d4a9b9d6..c9468c1106 100644 --- a/lang/lang.go +++ b/lang/lang.go @@ -21,13 +21,14 @@ package lang import ( "bytes" + "context" "fmt" "sync" "github.com/purpleidea/mgmt/engine" "github.com/purpleidea/mgmt/lang/ast" - "github.com/purpleidea/mgmt/lang/funcs" _ "github.com/purpleidea/mgmt/lang/funcs/core" // import so the funcs register + "github.com/purpleidea/mgmt/lang/funcs/dage" "github.com/purpleidea/mgmt/lang/funcs/vars" "github.com/purpleidea/mgmt/lang/inputs" "github.com/purpleidea/mgmt/lang/interfaces" @@ -62,30 +63,20 @@ type Lang struct { Logf func(format string, v ...interface{}) ast interfaces.Stmt // store main prog AST here - funcs *funcs.Engine // function event engine + funcs *dage.Engine // function event engine + graph *pgraph.Graph // function graph - loadedChan chan struct{} // loaded signal - - streamChan chan error // signals a new graph can be created or problem + streamChan <-chan error // signals a new graph can be created or problem //streamBurst bool // should we try and be bursty with the stream events? - closeChan chan struct{} // close signal - wg *sync.WaitGroup + wg *sync.WaitGroup } -// Init initializes the lang struct, and starts up the initial data sources. +// Init initializes the lang struct, and starts up the initial input parsing. // NOTE: The trick is that we need to get the list of funcs to watch AND start // watching them, *before* we pull their values, that way we'll know if they // changed from the values we wanted. func (obj *Lang) Init() error { - obj.loadedChan = make(chan struct{}) - obj.streamChan = make(chan error) - obj.closeChan = make(chan struct{}) - obj.wg = &sync.WaitGroup{} - - once := &sync.Once{} - loadedSignal := func() { close(obj.loadedChan) } // only run once! - if obj.Debug { obj.Logf("input: %s", obj.Input) tree, err := util.FsTree(obj.Fs, "/") // should look like gapi @@ -215,114 +206,120 @@ func (obj *Lang) Init() error { obj.Logf("building function graph...") // we assume that for some given code, the list of funcs doesn't change // iow, we don't support variable, variables or absurd things like that - graph, err := obj.ast.Graph() // build the graph of functions + obj.graph = &pgraph.Graph{Name: "functionGraph"} + env := make(map[string]interfaces.Func) + for k, v := range scope.Variables { + g, builtinFunc, err := v.Graph(nil) + if err != nil { + return errwrap.Wrapf(err, "calling Graph on builtins") + } + obj.graph.AddGraph(g) + env[k] = builtinFunc + } + g, err := obj.ast.Graph() // build the graph of functions if err != nil { return errwrap.Wrapf(err, "could not generate function graph") } + obj.graph.AddGraph(g) if obj.Debug { - obj.Logf("function graph: %+v", graph) - graph.Logf(obj.Logf) // log graph output with this logger... - } - - if graph.NumVertices() == 0 { // no funcs to load! - // send only one signal since we won't ever send after this! - obj.Logf("static graph found") - obj.wg.Add(1) - go func() { - defer obj.wg.Done() - defer close(obj.streamChan) // no more events are coming! - close(obj.loadedChan) // signal - select { - case obj.streamChan <- nil: // send one signal - // pass - case <-obj.closeChan: - return - } - }() - return nil // exit early, no funcs to load! + obj.Logf("function graph: %+v", obj.graph) + obj.graph.Logf(obj.Logf) // log graph output with this logger... + //if err := obj.graph.ExecGraphviz("/tmp/graphviz.dot"); err != nil { + // return errwrap.Wrapf(err, "writing graph failed") + //} } - obj.funcs = &funcs.Engine{ - Graph: graph, // not the same as the output graph! + obj.funcs = &dage.Engine{ + Name: "lang", // TODO: arbitrary name for now Hostname: obj.Hostname, World: obj.World, Debug: obj.Debug, Logf: func(format string, v ...interface{}) { obj.Logf("funcs: "+format, v...) }, - Glitch: false, // FIXME: verify this functionality is perfect! } obj.Logf("function engine initializing...") - if err := obj.funcs.Init(); err != nil { + if err := obj.funcs.Setup(); err != nil { return errwrap.Wrapf(err, "init error with func engine") } - obj.Logf("function engine validating...") - if err := obj.funcs.Validate(); err != nil { - return errwrap.Wrapf(err, "validate error with func engine") - } + obj.streamChan = obj.funcs.Stream() // after obj.funcs.Setup runs + + return nil +} + +// Run kicks off the function engine. Use the context to shut it down. +func (obj *Lang) Run(ctx context.Context) (reterr error) { + wg := &sync.WaitGroup{} + defer wg.Wait() + + runCtx, cancel := context.WithCancel(context.Background()) // Don't inherit from parent + defer cancel() + + //obj.Logf("function engine validating...") + //if err := obj.funcs.Validate(); err != nil { + // return errwrap.Wrapf(err, "validate error with func engine") + //} obj.Logf("function engine starting...") - // On failure, we expect the caller to run Close() to shutdown all of - // the currently initialized (and running) funcs... This is needed if - // we successfully ran `Run` but isn't needed only for Init/Validate. - if err := obj.funcs.Run(); err != nil { - return errwrap.Wrapf(err, "run error with func engine") + wg.Add(1) + go func() { + defer wg.Done() + if err := obj.funcs.Run(runCtx); err == nil { + reterr = errwrap.Append(reterr, err) + } + // Run() should only error if not a dag I think... + }() + + <-obj.funcs.Started() // wait for startup (will not block forever) + + // Sanity checks for graph size. + if count := obj.funcs.NumVertices(); count != 0 { + return fmt.Errorf("expected empty graph on start, got %d vertices", count) } + defer func() { + if count := obj.funcs.NumVertices(); count != 0 { + err := fmt.Errorf("expected empty graph on exit, got %d vertices", count) + reterr = errwrap.Append(reterr, err) + } + }() + defer wg.Wait() + defer cancel() // now cancel Run only after Reverse and Free are done! + + txn := obj.funcs.Txn() + defer txn.Free() // remember to call Free() + txn.AddGraph(obj.graph) + if err := txn.Commit(); err != nil { + return errwrap.Wrapf(err, "error adding to function graph engine") + } + defer func() { + if err := txn.Reverse(); err != nil { // should remove everything we added + reterr = errwrap.Append(reterr, err) + } + }() // wait for some activity obj.Logf("stream...") - stream := obj.funcs.Stream() - obj.wg.Add(1) - go func() { - obj.Logf("loop...") - defer obj.wg.Done() - defer close(obj.streamChan) // no more events are coming! - for { - var err error - var ok bool - select { - case err, ok = <-stream: - if !ok { - obj.Logf("stream closed") - return - } - if err == nil { - // only do this once, on the first event - once.Do(loadedSignal) // signal - } - - case <-obj.closeChan: - return - } - - select { - case obj.streamChan <- err: // send - if err != nil { - obj.Logf("stream error: %+v", err) - return - } - - case <-obj.closeChan: - return - } - } - }() + + select { + case <-ctx.Done(): + } + return nil } // Stream returns a channel of graph change requests or errors. These are // usually sent when a func output changes. -func (obj *Lang) Stream() chan error { +func (obj *Lang) Stream() <-chan error { return obj.streamChan } // Interpret runs the interpreter and returns a graph and corresponding error. func (obj *Lang) Interpret() (*pgraph.Graph, error) { select { - case <-obj.loadedChan: // funcs are now loaded! + case <-obj.funcs.Loaded(): // funcs are now loaded! // pass default: // if this is hit, someone probably called this too early! @@ -332,37 +329,9 @@ func (obj *Lang) Interpret() (*pgraph.Graph, error) { obj.Logf("running interpret...") table := obj.funcs.Table() // map[pgraph.Vertex]types.Value - fn := func(n interfaces.Node) error { - expr, ok := n.(interfaces.Expr) - if !ok { - return nil - } - v, ok := expr.(pgraph.Vertex) - if !ok { - panic("programming error in interfaces.Expr -> pgraph.Vertex lookup") - } - val, exists := table[v] - if !exists { - fmt.Printf("XXX: missing value in table is pointer: %p\n", v) - return nil // XXX: workaround for now... - //return fmt.Errorf("missing value in table for: %s", v) - } - return expr.SetValue(val) // set the value - } - obj.funcs.Lock() // XXX: apparently there are races between SetValue and reading obj.V values... - if err := obj.ast.Apply(fn); err != nil { - if obj.Debug { - for k, v := range table { - obj.Logf("table: key: %+v ; value: %+v", k, v) - } - } - obj.funcs.Unlock() - return nil, err - } - obj.funcs.Unlock() // this call returns the graph - graph, err := interpret.Interpret(obj.ast) + graph, err := interpret.Interpret(obj.ast, table) if err != nil { return nil, errwrap.Wrapf(err, "could not interpret") } @@ -370,14 +339,7 @@ func (obj *Lang) Interpret() (*pgraph.Graph, error) { return graph, nil // return a graph } -// Close shuts down the lang struct and causes all the funcs to shutdown. It -// must be called when finished after any successful Init ran. -func (obj *Lang) Close() error { - var err error - if obj.funcs != nil { - err = obj.funcs.Close() - } - close(obj.closeChan) - obj.wg.Wait() - return err +// Cleanup cleans up and frees memory and resources after everything is done. +func (obj *Lang) Cleanup() error { + return obj.funcs.Cleanup() } diff --git a/lang/lang_test.go b/lang/lang_test.go index d2b51f142c..026b5f79e7 100644 --- a/lang/lang_test.go +++ b/lang/lang_test.go @@ -20,7 +20,9 @@ package lang import ( + "context" "fmt" + "sync" "testing" "github.com/purpleidea/mgmt/engine" @@ -90,7 +92,7 @@ func edgeCmpFn(e1, e2 pgraph.Edge) (bool, error) { return e1.String() == e2.String(), nil } -func runInterpret(t *testing.T, code string) (*pgraph.Graph, error) { +func runInterpret(t *testing.T, code string) (_ *pgraph.Graph, reterr error) { logf := func(format string, v ...interface{}) { t.Logf("test: lang: "+format, v...) } @@ -123,31 +125,41 @@ func runInterpret(t *testing.T, code string) (*pgraph.Graph, error) { if err := lang.Init(); err != nil { return nil, errwrap.Wrapf(err, "init failed") } - closeFn := func() error { - return errwrap.Wrapf(lang.Close(), "close failed") - } + defer lang.Cleanup() + + wg := &sync.WaitGroup{} + defer wg.Wait() + + ctx, cancel := context.WithCancel(context.Background()) // TODO: get it from parent + defer cancel() + + wg.Add(1) + go func() { + defer wg.Done() + if err := lang.Run(ctx); err != nil { + reterr = errwrap.Append(reterr, err) + } + }() + defer cancel() // shutdown the Run // we only wait for the first event, instead of the continuous stream select { case err, ok := <-lang.Stream(): if !ok { - return nil, errwrap.Wrapf(closeFn(), "stream closed without event") + return nil, fmt.Errorf("stream closed without event") } if err != nil { - return nil, errwrap.Wrapf(err, "stream failed, close: %+v", closeFn()) + return nil, errwrap.Wrapf(err, "stream failed") } } // run artificially without the entire GAPI loop graph, err := lang.Interpret() if err != nil { - err := errwrap.Wrapf(err, "interpret failed") - e := closeFn() - err = errwrap.Append(err, e) // list of errors - return nil, err + return nil, errwrap.Wrapf(err, "interpret failed") } - return graph, closeFn() + return graph, nil } // TODO: empty code is not currently allowed, should we allow it? diff --git a/lang/types/full/full.go b/lang/types/full/full.go new file mode 100644 index 0000000000..cd67c1541e --- /dev/null +++ b/lang/types/full/full.go @@ -0,0 +1,144 @@ +// Mgmt +// Copyright (C) 2013-2023+ James Shubin and the project contributors +// Written by James Shubin and the project contributors +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package full + +import ( + "fmt" + + "github.com/purpleidea/mgmt/lang/interfaces" + "github.com/purpleidea/mgmt/lang/types" + "github.com/purpleidea/mgmt/util/errwrap" +) + +// FuncValue represents a function value, for example a built-in or a lambda. +// +// In most languages, we can simply call a function with a list of arguments and +// expect to receive a single value. In this language, however, a function might +// be something like datetime.now() or fn(n) {shell(Sprintf("seq %d", n))}, +// which might not produce a value immediately, and might then produce multiple +// values over time. Thus, in this language, a FuncValue does not receive +// Values, instead it receives input Func nodes. The FuncValue then adds more +// Func nodes and edges in order to arrange for output values to be sent to a +// particular output node, which the function returns so that the caller may +// connect that output node to more nodes down the line. +// +// The function can also return an error which could represent that something +// went horribly wrong. (Think, an internal panic.) +type FuncValue struct { + types.Base + V func(interfaces.Txn, []interfaces.Func) (interfaces.Func, error) + T *types.Type // contains ordered field types, arg names are a bonus part +} + +// NewFunc creates a new function with the specified type. +func NewFunc(t *types.Type) *FuncValue { + if t.Kind != types.KindFunc { + return nil // sanity check + } + v := func(interfaces.Txn, []interfaces.Func) (interfaces.Func, error) { + return nil, fmt.Errorf("nil function") // TODO: is this correct? + } + return &FuncValue{ + V: v, + T: t, + } +} + +// String returns a visual representation of this value. +func (obj *FuncValue) String() string { + return fmt.Sprintf("func(%+v)", obj.T) // TODO: can't print obj.V w/o vet warning +} + +// Type returns the type data structure that represents this type. +func (obj *FuncValue) Type() *types.Type { return obj.T } + +// Less compares to value and returns true if we're smaller. This panics if the +// two types aren't the same. +func (obj *FuncValue) Less(v types.Value) bool { + panic("functions are not comparable") +} + +// Cmp returns an error if this value isn't the same as the arg passed in. +func (obj *FuncValue) Cmp(val types.Value) error { + if obj == nil || val == nil { + return fmt.Errorf("cannot cmp to nil") + } + if err := obj.Type().Cmp(val.Type()); err != nil { + return errwrap.Wrapf(err, "cannot cmp types") + } + + if obj != val { // best we can do + return fmt.Errorf("different pointers") + } + + return nil +} + +// Copy returns a copy of this value. +func (obj *FuncValue) Copy() types.Value { + // TODO: can we do something useful here? + panic("cannot implement Copy() for FuncValue, because FuncValue is a full.FuncValue, not a Value") +} + +// Value returns the raw value of this type. +func (obj *FuncValue) Value() interface{} { + // TODO: can we do something useful here? + panic("cannot implement Value() for FuncValue, because FuncValue is a full.FuncValue, not a Value") + //typ := obj.T.Reflect() + // + //// wrap our function with the translation that is necessary + //fn := func(args []reflect.Value) (results []reflect.Value) { // build + // innerArgs := []types.Value{} + // for _, x := range args { + // v, err := types.ValueOf(x) // reflect.Value -> Value + // if err != nil { + // panic(fmt.Sprintf("can't determine value of %+v", x)) + // } + // innerArgs = append(innerArgs, v) + // } + // result, err := obj.V(innerArgs) // call it + // if err != nil { + // // when calling our function with the Call method, then + // // we get the error output and have a chance to decide + // // what to do with it, but when calling it from within + // // a normal golang function call, the error represents + // // that something went horribly wrong, aka a panic... + // panic(fmt.Sprintf("function panic: %+v", err)) + // } + // return []reflect.Value{reflect.ValueOf(result.Value())} // only one result + //} + //val := reflect.MakeFunc(typ, fn) + //return val.Interface() +} + +// Func represents the value of this type as a function if it is one. If this is +// not a function, then this panics. +func (obj *FuncValue) Func() interface{} { + return obj.V +} + +// Set sets the function value to be a new function. +func (obj *FuncValue) Set(fn func(interfaces.Txn, []interfaces.Func) (interfaces.Func, error)) error { // TODO: change method name? + obj.V = fn + return nil // TODO: can we do any sort of checking here? +} + +// Call calls the function with the provided txn and args. +func (obj *FuncValue) Call(txn interfaces.Txn, args []interfaces.Func) (interfaces.Func, error) { + return obj.V(txn, args) +} diff --git a/lang/types/value.go b/lang/types/value.go index 89adaa391a..3a6df6d3af 100644 --- a/lang/types/value.go +++ b/lang/types/value.go @@ -54,7 +54,7 @@ type Value interface { List() []Value Map() map[Value]Value // keys must all have same type, same for values Struct() map[string]Value - Func() func([]Value) (Value, error) + Func() interface{} // func(interfaces.Txn, []interfaces.Func) (interfaces.Func, error) } // ValueOfGolang is a helper that takes a golang value, and produces the mcl @@ -263,6 +263,9 @@ func Into(v Value, rv reflect.Value) error { } rv = rv.Elem() // un-nest rv from pointer } + if !rv.CanSet() { + return fmt.Errorf("can't set value, is it unexported?") + } // capture rv and v in a closure that is static for the scope of this Into() call // mustInto ensures rv is in a list of compatible types before attempting to reflect it @@ -389,7 +392,12 @@ func Into(v Value, rv reflect.Value) error { return err } + keys := []string{} for k := range v.T.Map { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { // loop in deterministic order mk := k // map mcl field name -> go field name based on `lang:""` tag if key, exists := mapping[k]; exists { @@ -402,40 +410,6 @@ func Into(v Value, rv reflect.Value) error { } return nil - case *FuncValue: - if err := mustInto(reflect.Func); err != nil { - return err - } - - // wrap our function with the translation that is necessary - fn := func(args []reflect.Value) (results []reflect.Value) { // build - innerArgs := []Value{} - for _, x := range args { - v, err := ValueOf(x) // reflect.Value -> Value - if err != nil { - panic(fmt.Errorf("can't determine value of %+v", x)) - } - innerArgs = append(innerArgs, v) - } - result, err := v.V(innerArgs) // call it - if err != nil { - // when calling our function with the Call method, then - // we get the error output and have a chance to decide - // what to do with it, but when calling it from within - // a normal golang function call, the error represents - // that something went horribly wrong, aka a panic... - panic(fmt.Errorf("function panic: %+v", err)) - } - out := reflect.New(rv.Type().Out(0)) - // convert the lang result back to a Go value - if err := Into(result, out); err != nil { - panic(fmt.Errorf("function return conversion panic: %+v", err)) - } - return []reflect.Value{out} // only one result - } - rv.Set(reflect.MakeFunc(rv.Type(), fn)) - return nil - case *VariantValue: return Into(v.V, rv) @@ -451,54 +425,54 @@ func (vs ValueSlice) Len() int { return len(vs) } func (vs ValueSlice) Swap(i, j int) { vs[i], vs[j] = vs[j], vs[i] } func (vs ValueSlice) Less(i, j int) bool { return vs[i].Less(vs[j]) } -// base implements the missing methods that all types need. -type base struct{} +// Base implements the missing methods that all types need. +type Base struct{} // Bool represents the value of this type as a bool if it is one. If this is not // a bool, then this panics. -func (obj *base) Bool() bool { +func (obj *Base) Bool() bool { panic("not a bool") } // Str represents the value of this type as a string if it is one. If this is // not a string, then this panics. -func (obj *base) Str() string { +func (obj *Base) Str() string { panic("not an str") // yes, i think this is the correct grammar } // Int represents the value of this type as an integer if it is one. If this is // not an integer, then this panics. -func (obj *base) Int() int64 { +func (obj *Base) Int() int64 { panic("not an int") } // Float represents the value of this type as a float if it is one. If this is // not a float, then this panics. -func (obj *base) Float() float64 { +func (obj *Base) Float() float64 { panic("not a float") } // List represents the value of this type as a list if it is one. If this is not // a list, then this panics. -func (obj *base) List() []Value { +func (obj *Base) List() []Value { panic("not a list") } // Map represents the value of this type as a dictionary if it is one. If this // is not a map, then this panics. -func (obj *base) Map() map[Value]Value { +func (obj *Base) Map() map[Value]Value { panic("not a list") } // Struct represents the value of this type as a struct if it is one. If this is // not a struct, then this panics. -func (obj *base) Struct() map[string]Value { +func (obj *Base) Struct() map[string]Value { panic("not a struct") } // Func represents the value of this type as a function if it is one. If this is // not a function, then this panics. -func (obj *base) Func() func([]Value) (Value, error) { +func (obj *Base) Func() interface{} { panic("not a func") } @@ -506,7 +480,7 @@ func (obj *base) Func() func([]Value) (Value, error) { // that this base implementation of the method be replaced in the specific type. // This *may* panic if the two types aren't the same. // NOTE: this can be used as an example template to write your own function. -//func (obj *base) Less(v Value) bool { +//func (obj *Base) Less(v Value) bool { // // TODO: cheap less, be smarter in each type eg: int's should cmp as int // return obj.String() < v.String() //} @@ -515,7 +489,7 @@ func (obj *base) Func() func([]Value) (Value, error) { // implementation uses the base Less implementation and should be replaced. It // is always nice to implement this properly so that we get better error output. // NOTE: this can be used as an example template to write your own function. -//func (obj *base) Cmp(v Value) error { +//func (obj *Base) Cmp(v Value) error { // // if they're both true or both false, then they must be the same, // // because we expect that if x < & && y < x then x == y // if obj.Less(v) != v.Less(obj) { @@ -526,7 +500,7 @@ func (obj *base) Func() func([]Value) (Value, error) { // BoolValue represents a boolean value. type BoolValue struct { - base + Base V bool } @@ -589,7 +563,7 @@ func (obj *BoolValue) Bool() bool { // StrValue represents a string value. type StrValue struct { - base + Base V string } @@ -644,7 +618,7 @@ func (obj *StrValue) Str() string { // IntValue represents an integer value. type IntValue struct { - base + Base V int64 } @@ -698,7 +672,7 @@ func (obj *IntValue) Int() int64 { // FloatValue represents an integer value. type FloatValue struct { - base + Base V float64 } @@ -755,7 +729,7 @@ func (obj *FloatValue) Float() float64 { // ListValue represents a list value. type ListValue struct { - base + Base V []Value // all elements must have type T.Val T *Type } @@ -883,7 +857,7 @@ func (obj *ListValue) Contains(v Value) (index int, exists bool) { // MapValue represents a dictionary value. type MapValue struct { - base + Base // the types of all keys and values are represented inside of T V map[Value]Value T *Type @@ -1016,7 +990,7 @@ func (obj *MapValue) Lookup(key Value) (value Value, exists bool) { // StructValue represents a struct value. The keys are ordered. // TODO: if all functions require arg names to call, we don't need to order! type StructValue struct { - base + Base V map[string]Value // each field can have a different type T *Type // contains ordered field types } @@ -1128,23 +1102,33 @@ func (obj *StructValue) Lookup(k string) (value Value, exists bool) { return v, exists } -// FuncValue represents a function value. The defined function takes a list of -// Value arguments and returns a Value. It can also return an error which could -// represent that something went horribly wrong. (Think, an internal panic.) +// FuncValue represents a function which takes a list of Value arguments and +// returns a Value. It can also return an error which could represent that +// something went horribly wrong. (Think, an internal panic.) +// +// This is not general enough to represent all functions in the language (see +// the full.FuncValue), but it is a useful common case. +// +// FuncValue is not a Value, but it is a useful building block for implementing +// Func nodes. type FuncValue struct { - base + Base V func([]Value) (Value, error) T *Type // contains ordered field types, arg names are a bonus part } -// NewFunc creates a new function with the specified type. +// NewFunc creates a useless function which will get overwritten by something +// more useful later. func NewFunc(t *Type) *FuncValue { if t.Kind != KindFunc { return nil // sanity check } v := func([]Value) (Value, error) { - return nil, fmt.Errorf("nil function") // TODO: is this correct? + // You were not supposed to call the temporary function, you + // were supposed to replace it with a real implementation! + return nil, fmt.Errorf("nil function") } + // return an empty interface{} return &FuncValue{ V: v, T: t, @@ -1156,77 +1140,6 @@ func (obj *FuncValue) String() string { return fmt.Sprintf("func(%+v)", obj.T) // TODO: can't print obj.V w/o vet warning } -// Type returns the type data structure that represents this type. -func (obj *FuncValue) Type() *Type { return obj.T } - -// Less compares to value and returns true if we're smaller. This panics if the -// two types aren't the same. -func (obj *FuncValue) Less(v Value) bool { - V := v.(*FuncValue) - return obj.String() < V.String() // FIXME: implement a proper less func -} - -// Cmp returns an error if this value isn't the same as the arg passed in. -func (obj *FuncValue) Cmp(val Value) error { - if obj == nil || val == nil { - return fmt.Errorf("cannot cmp to nil") - } - if err := obj.Type().Cmp(val.Type()); err != nil { - return errwrap.Wrapf(err, "cannot cmp types") - } - - return fmt.Errorf("cannot cmp funcs") // TODO: can we ? -} - -// Copy returns a copy of this value. -func (obj *FuncValue) Copy() Value { - return &FuncValue{ - V: obj.V, // FIXME: can we copy the function, or do we need to? - T: obj.T.Copy(), - } -} - -// Value returns the raw value of this type. -func (obj *FuncValue) Value() interface{} { - typ := obj.T.Reflect() - - // wrap our function with the translation that is necessary - fn := func(args []reflect.Value) (results []reflect.Value) { // build - innerArgs := []Value{} - for _, x := range args { - v, err := ValueOf(x) // reflect.Value -> Value - if err != nil { - panic(fmt.Sprintf("can't determine value of %+v", x)) - } - innerArgs = append(innerArgs, v) - } - result, err := obj.V(innerArgs) // call it - if err != nil { - // when calling our function with the Call method, then - // we get the error output and have a chance to decide - // what to do with it, but when calling it from within - // a normal golang function call, the error represents - // that something went horribly wrong, aka a panic... - panic(fmt.Sprintf("function panic: %+v", err)) - } - return []reflect.Value{reflect.ValueOf(result.Value())} // only one result - } - val := reflect.MakeFunc(typ, fn) - return val.Interface() -} - -// Func represents the value of this type as a function if it is one. If this is -// not a function, then this panics. -func (obj *FuncValue) Func() func([]Value) (Value, error) { - return obj.V -} - -// Set sets the function value to be a new function. -func (obj *FuncValue) Set(fn func([]Value) (Value, error)) error { // TODO: change method name? - obj.V = fn - return nil // TODO: can we do any sort of checking here? -} - // Call runs the function value and returns its result. It returns an error if // something goes wrong during execution, and panic's if you call this with // inappropriate input types, or if it returns an inappropriate output type. @@ -1256,9 +1169,38 @@ func (obj *FuncValue) Call(args []Value) (Value, error) { return result, err } +// Type returns the type data structure that represents this type. +func (obj *FuncValue) Type() *Type { return obj.T } + +// Less compares to value and returns true if we're smaller. This panics if the +// two types aren't the same. In this situation, they can't be compared so we +// panic. +func (obj *FuncValue) Less(v Value) bool { + panic("you cannot compare functions") +} + +// Cmp returns an error if this value isn't the same as the arg passed in. In +// this situation, they can't be compared so we panic. +func (obj *FuncValue) Cmp(val Value) error { + panic("you cannot compare functions") +} + +// Copy returns a copy of this value. +func (obj *FuncValue) Copy() Value { + return &FuncValue{ + V: obj.V, + T: obj.T.Copy(), + } +} + +// Value returns the raw value of this type. +func (obj *FuncValue) Value() interface{} { + return obj.V +} + // VariantValue represents a variant value. type VariantValue struct { - base + Base V Value // formerly I experimented with using interface{} instead T *Type } @@ -1377,6 +1319,6 @@ func (obj *VariantValue) Struct() map[string]Value { // Func represents the value of this type as a function if it is one. If this is // not a function, then this panics. -func (obj *VariantValue) Func() func([]Value) (Value, error) { +func (obj *VariantValue) Func() interface{} { return obj.V.Func() } diff --git a/lang/unification_test.go b/lang/unification_test.go index f8c9bff122..f147448c11 100644 --- a/lang/unification_test.go +++ b/lang/unification_test.go @@ -451,60 +451,6 @@ func TestUnification1(t *testing.T) { }, }) } - { - //$v = 42 - //$x = template("hello", $v) # redirect var for harder unification - //test "t1" { - // anotherstr => $x, - //} - innerFunc := &ast.ExprCall{ - Name: "template", - Args: []interfaces.Expr{ - &ast.ExprStr{ - V: "hello", // whatever... - }, - &ast.ExprVar{ - Name: "v", - }, - }, - } - stmt := &ast.StmtProg{ - Body: []interfaces.Stmt{ - &ast.StmtBind{ - Ident: "v", - Value: &ast.ExprInt{ - V: 42, - }, - }, - &ast.StmtBind{ - Ident: "x", - Value: innerFunc, - }, - &ast.StmtRes{ - Kind: "test", - Name: &ast.ExprStr{ - V: "t1", - }, - Contents: []ast.StmtResContents{ - &ast.StmtResField{ - Field: "anotherstr", - Value: &ast.ExprVar{ - Name: "x", - }, - }, - }, - }, - }, - } - testCases = append(testCases, test{ - name: "complex template", - ast: stmt, - fail: false, - expect: map[interfaces.Expr]*types.Type{ - innerFunc: types.NewType("str"), - }, - }) - } { // import "datetime" //test "t1" { diff --git a/misc/txtar-port.sh b/misc/txtar-port.sh index b1902c975f..016fb2f047 100644 --- a/misc/txtar-port.sh +++ b/misc/txtar-port.sh @@ -5,3 +5,9 @@ for f in */main.mcl; do cat $f | cat - $(dirname $f).txtar | sponge $(dirname $f).txtar echo '-- main.mcl --' | cat - $(dirname $f).txtar | sponge $(dirname $f).txtar done + +#for f in *.txtar; do +# echo $f +# #cat $f | cat - $(dirname $f).txtar | sponge $(dirname $f).txtar +# echo '-- OUTPUT --' | cat - $f | sponge $f +#done