From 32dd9fd086e7021bc95fda0ecce96ec718b257eb Mon Sep 17 00:00:00 2001 From: sam boyer Date: Mon, 18 Dec 2017 09:06:21 -0500 Subject: [PATCH 1/3] dep: update go-toml to latest master --- Gopkg.lock | 8 +- .../pelletier/go-buffruneio/.gitignore | 1 - .../pelletier/go-buffruneio/.travis.yml | 7 - .../pelletier/go-buffruneio/README.md | 62 --- .../pelletier/go-buffruneio/buffruneio.go | 117 ----- .../go-buffruneio/buffruneio_test.go | 145 ------ .../github.com/pelletier/go-toml/.gitignore | 1 + .../github.com/pelletier/go-toml/.travis.yml | 7 +- vendor/github.com/pelletier/go-toml/README.md | 101 ++-- .../pelletier/go-toml/benchmark.json | 164 ++++++ .../github.com/pelletier/go-toml/benchmark.sh | 32 ++ .../pelletier/go-toml/benchmark.toml | 244 +++++++++ .../pelletier/go-toml/benchmark.yml | 121 +++++ .../pelletier/go-toml/benchmark_test.go | 192 +++++++ vendor/github.com/pelletier/go-toml/clean.sh | 6 - vendor/github.com/pelletier/go-toml/doc.go | 251 +-------- .../github.com/pelletier/go-toml/doc_test.go | 136 +++-- vendor/github.com/pelletier/go-toml/fuzz.go | 31 ++ vendor/github.com/pelletier/go-toml/fuzz.sh | 15 + .../pelletier/go-toml/keysparsing.go | 107 +++- .../pelletier/go-toml/keysparsing_test.go | 16 +- vendor/github.com/pelletier/go-toml/lexer.go | 88 ++-- .../pelletier/go-toml/lexer_test.go | 54 +- .../github.com/pelletier/go-toml/marshal.go | 325 +++++++++--- .../pelletier/go-toml/marshal_test.go | 223 ++++++++ vendor/github.com/pelletier/go-toml/match.go | 234 --------- .../pelletier/go-toml/match_test.go | 201 -------- vendor/github.com/pelletier/go-toml/parser.go | 72 ++- .../pelletier/go-toml/parser_test.go | 14 +- vendor/github.com/pelletier/go-toml/query.go | 153 ------ .../pelletier/go-toml/query_test.go | 70 --- .../pelletier/go-toml/querylexer.go | 356 ------------- .../pelletier/go-toml/querylexer_test.go | 178 ------- .../pelletier/go-toml/queryparser.go | 275 ---------- .../pelletier/go-toml/queryparser_test.go | 483 ------------------ vendor/github.com/pelletier/go-toml/test.sh | 10 +- vendor/github.com/pelletier/go-toml/toml.go | 144 +++--- .../github.com/pelletier/go-toml/toml_test.go | 29 +- .../pelletier/go-toml/tomltree_create.go | 31 +- .../pelletier/go-toml/tomltree_create_test.go | 24 +- .../pelletier/go-toml/tomltree_write.go | 178 +++++-- .../pelletier/go-toml/tomltree_write_test.go | 151 ++++-- 42 files changed, 1995 insertions(+), 3062 deletions(-) delete mode 100644 vendor/github.com/pelletier/go-buffruneio/.gitignore delete mode 100644 vendor/github.com/pelletier/go-buffruneio/.travis.yml delete mode 100644 vendor/github.com/pelletier/go-buffruneio/README.md delete mode 100644 vendor/github.com/pelletier/go-buffruneio/buffruneio.go delete mode 100644 vendor/github.com/pelletier/go-buffruneio/buffruneio_test.go create mode 100644 vendor/github.com/pelletier/go-toml/benchmark.json create mode 100755 vendor/github.com/pelletier/go-toml/benchmark.sh create mode 100644 vendor/github.com/pelletier/go-toml/benchmark.toml create mode 100644 vendor/github.com/pelletier/go-toml/benchmark.yml create mode 100644 vendor/github.com/pelletier/go-toml/benchmark_test.go delete mode 100755 vendor/github.com/pelletier/go-toml/clean.sh create mode 100644 vendor/github.com/pelletier/go-toml/fuzz.go create mode 100755 vendor/github.com/pelletier/go-toml/fuzz.sh delete mode 100644 vendor/github.com/pelletier/go-toml/match.go delete mode 100644 vendor/github.com/pelletier/go-toml/match_test.go delete mode 100644 vendor/github.com/pelletier/go-toml/query.go delete mode 100644 vendor/github.com/pelletier/go-toml/query_test.go delete mode 100644 vendor/github.com/pelletier/go-toml/querylexer.go delete mode 100644 vendor/github.com/pelletier/go-toml/querylexer_test.go delete mode 100644 vendor/github.com/pelletier/go-toml/queryparser.go delete mode 100644 vendor/github.com/pelletier/go-toml/queryparser_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 835c753e86..96d83a6929 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -50,17 +50,11 @@ packages = ["."] revision = "e83dc5e7bba095e8d32fb2124714bf41f2a30cb5" -[[projects]] - name = "github.com/pelletier/go-buffruneio" - packages = ["."] - revision = "c37440a7cf42ac63b919c752ca73a85067e05992" - version = "v0.2.0" - [[projects]] branch = "master" name = "github.com/pelletier/go-toml" packages = ["."] - revision = "fe206efb84b2bc8e8cfafe6b4c1826622be969e3" + revision = "b8b5e7696574464b2f9bf303a7b37781bb52889f" [[projects]] name = "github.com/pkg/errors" diff --git a/vendor/github.com/pelletier/go-buffruneio/.gitignore b/vendor/github.com/pelletier/go-buffruneio/.gitignore deleted file mode 100644 index c56069fe26..0000000000 --- a/vendor/github.com/pelletier/go-buffruneio/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.test \ No newline at end of file diff --git a/vendor/github.com/pelletier/go-buffruneio/.travis.yml b/vendor/github.com/pelletier/go-buffruneio/.travis.yml deleted file mode 100644 index 9720442cd8..0000000000 --- a/vendor/github.com/pelletier/go-buffruneio/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -sudo: false -go: - - 1.3.3 - - 1.4.3 - - 1.5.3 - - tip diff --git a/vendor/github.com/pelletier/go-buffruneio/README.md b/vendor/github.com/pelletier/go-buffruneio/README.md deleted file mode 100644 index ff608b3ab8..0000000000 --- a/vendor/github.com/pelletier/go-buffruneio/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# buffruneio - -[![Tests Status](https://travis-ci.org/pelletier/go-buffruneio.svg?branch=master)](https://travis-ci.org/pelletier/go-buffruneio) -[![GoDoc](https://godoc.org/github.com/pelletier/go-buffruneio?status.svg)](https://godoc.org/github.com/pelletier/go-buffruneio) - -Buffruneio is a wrapper around bufio to provide buffered runes access with -unlimited unreads. - -```go -import "github.com/pelletier/go-buffruneio" -``` - -## Examples - -```go -import ( - "fmt" - "github.com/pelletier/go-buffruneio" - "strings" -) - -reader := buffruneio.NewReader(strings.NewReader("abcd")) -fmt.Println(reader.ReadRune()) // 'a' -fmt.Println(reader.ReadRune()) // 'b' -fmt.Println(reader.ReadRune()) // 'c' -reader.UnreadRune() -reader.UnreadRune() -fmt.Println(reader.ReadRune()) // 'b' -fmt.Println(reader.ReadRune()) // 'c' -``` - -## Documentation - -The documentation and additional examples are available at -[godoc.org](http://godoc.org/github.com/pelletier/go-buffruneio). - -## Contribute - -Feel free to report bugs and patches using GitHub's pull requests system on -[pelletier/go-toml](https://github.com/pelletier/go-buffruneio). Any feedback is -much appreciated! - -## LICENSE - -Copyright (c) 2016 Thomas Pelletier - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/pelletier/go-buffruneio/buffruneio.go b/vendor/github.com/pelletier/go-buffruneio/buffruneio.go deleted file mode 100644 index 4e6d6ea610..0000000000 --- a/vendor/github.com/pelletier/go-buffruneio/buffruneio.go +++ /dev/null @@ -1,117 +0,0 @@ -// Package buffruneio is a wrapper around bufio to provide buffered runes access with unlimited unreads. -package buffruneio - -import ( - "bufio" - "container/list" - "errors" - "io" -) - -// Rune to indicate end of file. -const ( - EOF = -(iota + 1) -) - -// ErrNoRuneToUnread is returned by UnreadRune() when the read index is already at the beginning of the buffer. -var ErrNoRuneToUnread = errors.New("no rune to unwind") - -// Reader implements runes buffering for an io.Reader object. -type Reader struct { - buffer *list.List - current *list.Element - input *bufio.Reader -} - -// NewReader returns a new Reader. -func NewReader(rd io.Reader) *Reader { - return &Reader{ - buffer: list.New(), - input: bufio.NewReader(rd), - } -} - -type runeWithSize struct { - r rune - size int -} - -func (rd *Reader) feedBuffer() error { - r, size, err := rd.input.ReadRune() - - if err != nil { - if err != io.EOF { - return err - } - r = EOF - } - - newRuneWithSize := runeWithSize{r, size} - - rd.buffer.PushBack(newRuneWithSize) - if rd.current == nil { - rd.current = rd.buffer.Back() - } - return nil -} - -// ReadRune reads the next rune from buffer, or from the underlying reader if needed. -func (rd *Reader) ReadRune() (rune, int, error) { - if rd.current == rd.buffer.Back() || rd.current == nil { - err := rd.feedBuffer() - if err != nil { - return EOF, 0, err - } - } - - runeWithSize := rd.current.Value.(runeWithSize) - rd.current = rd.current.Next() - return runeWithSize.r, runeWithSize.size, nil -} - -// UnreadRune pushes back the previously read rune in the buffer, extending it if needed. -func (rd *Reader) UnreadRune() error { - if rd.current == rd.buffer.Front() { - return ErrNoRuneToUnread - } - if rd.current == nil { - rd.current = rd.buffer.Back() - } else { - rd.current = rd.current.Prev() - } - return nil -} - -// Forget removes runes stored before the current stream position index. -func (rd *Reader) Forget() { - if rd.current == nil { - rd.current = rd.buffer.Back() - } - for ; rd.current != rd.buffer.Front(); rd.buffer.Remove(rd.current.Prev()) { - } -} - -// PeekRune returns at most the next n runes, reading from the uderlying source if -// needed. Does not move the current index. It includes EOF if reached. -func (rd *Reader) PeekRunes(n int) []rune { - res := make([]rune, 0, n) - cursor := rd.current - for i := 0; i < n; i++ { - if cursor == nil { - err := rd.feedBuffer() - if err != nil { - return res - } - cursor = rd.buffer.Back() - } - if cursor != nil { - r := cursor.Value.(runeWithSize).r - res = append(res, r) - if r == EOF { - return res - } - cursor = cursor.Next() - } - } - return res -} diff --git a/vendor/github.com/pelletier/go-buffruneio/buffruneio_test.go b/vendor/github.com/pelletier/go-buffruneio/buffruneio_test.go deleted file mode 100644 index 67b0cba9b3..0000000000 --- a/vendor/github.com/pelletier/go-buffruneio/buffruneio_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package buffruneio - -import ( - "runtime/debug" - "strings" - "testing" -) - -func assertNoError(t *testing.T, err error) { - if err != nil { - t.Log("unexpected error", err) - debug.PrintStack() - t.FailNow() - } -} - -func assumeRunesArray(t *testing.T, expected []rune, got []rune) { - if len(expected) != len(got) { - t.Fatal("expected", len(expected), "runes, but got", len(got)) - } - for i := 0; i < len(got); i++ { - if expected[i] != got[i] { - t.Fatal("expected rune", expected[i], "at index", i, "but got", got[i]) - } - } -} - -func assumeRune(t *testing.T, rd *Reader, r rune) { - gotRune, size, err := rd.ReadRune() - assertNoError(t, err) - if gotRune != r { - t.Fatal("got", string(gotRune), - "(", []byte(string(gotRune)), ")", - "expected", string(r), - "(", []byte(string(r)), ")") - t.Fatal("got size", size, - "expected", len([]byte(string(r)))) - } -} - -func TestReadString(t *testing.T) { - s := "hello" - rd := NewReader(strings.NewReader(s)) - - assumeRune(t, rd, 'h') - assumeRune(t, rd, 'e') - assumeRune(t, rd, 'l') - assumeRune(t, rd, 'l') - assumeRune(t, rd, 'o') - assumeRune(t, rd, EOF) -} - -func TestMultipleEOF(t *testing.T) { - s := "" - rd := NewReader(strings.NewReader(s)) - - assumeRune(t, rd, EOF) - assumeRune(t, rd, EOF) -} - -func TestUnread(t *testing.T) { - s := "ab" - rd := NewReader(strings.NewReader(s)) - - assumeRune(t, rd, 'a') - assumeRune(t, rd, 'b') - assertNoError(t, rd.UnreadRune()) - assumeRune(t, rd, 'b') - assumeRune(t, rd, EOF) -} - -func TestUnreadEOF(t *testing.T) { - s := "" - rd := NewReader(strings.NewReader(s)) - - _ = rd.UnreadRune() - assumeRune(t, rd, EOF) - assumeRune(t, rd, EOF) - assertNoError(t, rd.UnreadRune()) - assumeRune(t, rd, EOF) -} - -func TestForget(t *testing.T) { - s := "hello" - rd := NewReader(strings.NewReader(s)) - - assumeRune(t, rd, 'h') - assumeRune(t, rd, 'e') - assumeRune(t, rd, 'l') - assumeRune(t, rd, 'l') - rd.Forget() - if rd.UnreadRune() != ErrNoRuneToUnread { - t.Fatal("no rune should be available") - } -} - -func TestForgetEmpty(t *testing.T) { - s := "" - rd := NewReader(strings.NewReader(s)) - - rd.Forget() - assumeRune(t, rd, EOF) - rd.Forget() -} - -func TestPeekEmpty(t *testing.T) { - s := "" - rd := NewReader(strings.NewReader(s)) - - runes := rd.PeekRunes(1) - if len(runes) != 1 { - t.Fatal("incorrect number of runes", len(runes)) - } - if runes[0] != EOF { - t.Fatal("incorrect rune", runes[0]) - } -} - -func TestPeek(t *testing.T) { - s := "a" - rd := NewReader(strings.NewReader(s)) - - runes := rd.PeekRunes(1) - assumeRunesArray(t, []rune{'a'}, runes) - - runes = rd.PeekRunes(1) - assumeRunesArray(t, []rune{'a'}, runes) - - assumeRune(t, rd, 'a') - runes = rd.PeekRunes(1) - assumeRunesArray(t, []rune{EOF}, runes) - - assumeRune(t, rd, EOF) -} - -func TestPeekLarge(t *testing.T) { - s := "abcdefg" - rd := NewReader(strings.NewReader(s)) - - runes := rd.PeekRunes(100) - if len(runes) != len(s)+1 { - t.Fatal("incorrect number of runes", len(runes)) - } - assumeRunesArray(t, []rune{'a', 'b', 'c', 'd', 'e', 'f', 'g', EOF}, runes) -} diff --git a/vendor/github.com/pelletier/go-toml/.gitignore b/vendor/github.com/pelletier/go-toml/.gitignore index f1b619018e..99e38bbc53 100644 --- a/vendor/github.com/pelletier/go-toml/.gitignore +++ b/vendor/github.com/pelletier/go-toml/.gitignore @@ -1 +1,2 @@ test_program/test_program_bin +fuzz/ diff --git a/vendor/github.com/pelletier/go-toml/.travis.yml b/vendor/github.com/pelletier/go-toml/.travis.yml index 64f03809a1..6e644fdfd5 100644 --- a/vendor/github.com/pelletier/go-toml/.travis.yml +++ b/vendor/github.com/pelletier/go-toml/.travis.yml @@ -1,16 +1,17 @@ sudo: false language: go go: - - 1.6.4 - - 1.7.5 - - 1.8 + - 1.8.4 + - 1.9.1 - tip matrix: allow_failures: - go: tip fast_finish: true script: + - if [ -n "$(go fmt ./...)" ]; then exit 1; fi - ./test.sh + - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git before_install: - go get github.com/axw/gocov/gocov - go get github.com/mattn/goveralls diff --git a/vendor/github.com/pelletier/go-toml/README.md b/vendor/github.com/pelletier/go-toml/README.md index b8137e022d..0d357acf35 100644 --- a/vendor/github.com/pelletier/go-toml/README.md +++ b/vendor/github.com/pelletier/go-toml/README.md @@ -16,65 +16,64 @@ This library supports TOML version Go-toml provides the following features for using data parsed from TOML documents: * Load TOML documents from files and string data -* Easily navigate TOML structure using TomlTree +* Easily navigate TOML structure using Tree +* Mashaling and unmarshaling to and from data structures * Line & column position data for all parsed elements -* Query support similar to JSON-Path +* [Query support similar to JSON-Path](query/) * Syntax errors contain line and column numbers -Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing: - -* Semantic evaluation of parsed TOML -* Informing a user of mistakes in the source document, after it has been parsed -* Programatic handling of default values on a case-by-case basis -* Using a TOML document as a flexible data-store - ## Import - import "github.com/pelletier/go-toml" - -## Usage +```go +import "github.com/pelletier/go-toml" +``` -### Example +## Usage example -Say you have a TOML file that looks like this: +Read a TOML document: -```toml +```go +config, _ := toml.Load(` [postgres] user = "pelletier" -password = "mypassword" +password = "mypassword"`) +// retrieve data directly +user := config.Get("postgres.user").(string) + +// or using an intermediate object +postgresConfig := config.Get("postgres").(*toml.Tree) +password := postgresConfig.Get("password").(string) ``` -Read the username and password like this: +Or use Unmarshal: ```go -import ( - "fmt" - "github.com/pelletier/go-toml" -) - -config, err := toml.LoadFile("config.toml") -if err != nil { - fmt.Println("Error ", err.Error()) -} else { - // retrieve data directly - user := config.Get("postgres.user").(string) - password := config.Get("postgres.password").(string) - - // or using an intermediate object - configTree := config.Get("postgres").(*toml.TomlTree) - user = configTree.Get("user").(string) - password = configTree.Get("password").(string) - fmt.Println("User is ", user, ". Password is ", password) - - // show where elements are in the file - fmt.Println("User position: %v", configTree.GetPosition("user")) - fmt.Println("Password position: %v", configTree.GetPosition("password")) - - // use a query to gather elements without walking the tree - results, _ := config.Query("$..[user,password]") - for ii, item := range results.Values() { - fmt.Println("Query result %d: %v", ii, item) - } +type Postgres struct { + User string + Password string +} +type Config struct { + Postgres Postgres +} + +doc := []byte(` +[Postgres] +User = "pelletier" +Password = "mypassword"`) + +config := Config{} +toml.Unmarshal(doc, &config) +fmt.Println("user=", config.Postgres.User) +``` + +Or use a query: + +```go +// use a query to gather elements without walking the tree +q, _ := query.Compile("$..[user,password]") +results := q.Execute(config) +for ii, item := range results.Values() { + fmt.Println("Query result %d: %v", ii, item) } ``` @@ -115,6 +114,18 @@ You have to make sure two kind of tests run: You can run both of them using `./test.sh`. +### Fuzzing + +The script `./fuzz.sh` is available to +run [go-fuzz](https://github.com/dvyukov/go-fuzz) on go-toml. + +## Versioning + +Go-toml follows [Semantic Versioning](http://semver.org/). The supported version +of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of +this document. The last two major versions of Go are supported +(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)). + ## License The MIT License (MIT). Read [LICENSE](LICENSE). diff --git a/vendor/github.com/pelletier/go-toml/benchmark.json b/vendor/github.com/pelletier/go-toml/benchmark.json new file mode 100644 index 0000000000..86f99c6a87 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.json @@ -0,0 +1,164 @@ +{ + "array": { + "key1": [ + 1, + 2, + 3 + ], + "key2": [ + "red", + "yellow", + "green" + ], + "key3": [ + [ + 1, + 2 + ], + [ + 3, + 4, + 5 + ] + ], + "key4": [ + [ + 1, + 2 + ], + [ + "a", + "b", + "c" + ] + ], + "key5": [ + 1, + 2, + 3 + ], + "key6": [ + 1, + 2 + ] + }, + "boolean": { + "False": false, + "True": true + }, + "datetime": { + "key1": "1979-05-27T07:32:00Z", + "key2": "1979-05-27T00:32:00-07:00", + "key3": "1979-05-27T00:32:00.999999-07:00" + }, + "float": { + "both": { + "key": 6.626e-34 + }, + "exponent": { + "key1": 5e+22, + "key2": 1000000, + "key3": -0.02 + }, + "fractional": { + "key1": 1, + "key2": 3.1415, + "key3": -0.01 + }, + "underscores": { + "key1": 9224617.445991227, + "key2": 1e+100 + } + }, + "fruit": [{ + "name": "apple", + "physical": { + "color": "red", + "shape": "round" + }, + "variety": [{ + "name": "red delicious" + }, + { + "name": "granny smith" + } + ] + }, + { + "name": "banana", + "variety": [{ + "name": "plantain" + }] + } + ], + "integer": { + "key1": 99, + "key2": 42, + "key3": 0, + "key4": -17, + "underscores": { + "key1": 1000, + "key2": 5349221, + "key3": 12345 + } + }, + "products": [{ + "name": "Hammer", + "sku": 738594937 + }, + {}, + { + "color": "gray", + "name": "Nail", + "sku": 284758393 + } + ], + "string": { + "basic": { + "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + }, + "literal": { + "multiline": { + "lines": "The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", + "regex2": "I [dw]on't need \\d{2} apples" + }, + "quoted": "Tom \"Dubs\" Preston-Werner", + "regex": "\u003c\\i\\c*\\s*\u003e", + "winpath": "C:\\Users\\nodejs\\templates", + "winpath2": "\\\\ServerX\\admin$\\system32\\" + }, + "multiline": { + "continued": { + "key1": "The quick brown fox jumps over the lazy dog.", + "key2": "The quick brown fox jumps over the lazy dog.", + "key3": "The quick brown fox jumps over the lazy dog." + }, + "key1": "One\nTwo", + "key2": "One\nTwo", + "key3": "One\nTwo" + } + }, + "table": { + "inline": { + "name": { + "first": "Tom", + "last": "Preston-Werner" + }, + "point": { + "x": 1, + "y": 2 + } + }, + "key": "value", + "subtable": { + "key": "another value" + } + }, + "x": { + "y": { + "z": { + "w": {} + } + } + } +} diff --git a/vendor/github.com/pelletier/go-toml/benchmark.sh b/vendor/github.com/pelletier/go-toml/benchmark.sh new file mode 100755 index 0000000000..8b8bb528e7 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +reference_ref=${1:-master} +reference_git=${2:-.} + +if ! `hash benchstat 2>/dev/null`; then + echo "Installing benchstat" + go get golang.org/x/perf/cmd/benchstat + go install golang.org/x/perf/cmd/benchstat +fi + +tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX` +ref_tempdir="${tempdir}/ref" +ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt" +local_benchmark="`pwd`/benchmark-local.txt" + +echo "=== ${reference_ref} (${ref_tempdir})" +git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null +pushd ${ref_tempdir} >/dev/null +git checkout ${reference_ref} >/dev/null 2>/dev/null +go test -bench=. -benchmem | tee ${ref_benchmark} +popd >/dev/null + +echo "" +echo "=== local" +go test -bench=. -benchmem | tee ${local_benchmark} + +echo "" +echo "=== diff" +benchstat -delta-test=none ${ref_benchmark} ${local_benchmark} \ No newline at end of file diff --git a/vendor/github.com/pelletier/go-toml/benchmark.toml b/vendor/github.com/pelletier/go-toml/benchmark.toml new file mode 100644 index 0000000000..dfd77e0962 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.toml @@ -0,0 +1,244 @@ +################################################################################ +## Comment + +# Speak your mind with the hash symbol. They go from the symbol to the end of +# the line. + + +################################################################################ +## Table + +# Tables (also known as hash tables or dictionaries) are collections of +# key/value pairs. They appear in square brackets on a line by themselves. + +[table] + +key = "value" # Yeah, you can do this. + +# Nested tables are denoted by table names with dots in them. Name your tables +# whatever crap you please, just don't use #, ., [ or ]. + +[table.subtable] + +key = "another value" + +# You don't need to specify all the super-tables if you don't want to. TOML +# knows how to do it for you. + +# [x] you +# [x.y] don't +# [x.y.z] need these +[x.y.z.w] # for this to work + + +################################################################################ +## Inline Table + +# Inline tables provide a more compact syntax for expressing tables. They are +# especially useful for grouped data that can otherwise quickly become verbose. +# Inline tables are enclosed in curly braces `{` and `}`. No newlines are +# allowed between the curly braces unless they are valid within a value. + +[table.inline] + +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } + + +################################################################################ +## String + +# There are four ways to express strings: basic, multi-line basic, literal, and +# multi-line literal. All strings must contain only valid UTF-8 characters. + +[string.basic] + +basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + +[string.multiline] + +# The following strings are byte-for-byte equivalent: +key1 = "One\nTwo" +key2 = """One\nTwo""" +key3 = """ +One +Two""" + +[string.multiline.continued] + +# The following strings are byte-for-byte equivalent: +key1 = "The quick brown fox jumps over the lazy dog." + +key2 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + +key3 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ + +[string.literal] + +# What you see is what you get. +winpath = 'C:\Users\nodejs\templates' +winpath2 = '\\ServerX\admin$\system32\' +quoted = 'Tom "Dubs" Preston-Werner' +regex = '<\i\c*\s*>' + + +[string.literal.multiline] + +regex2 = '''I [dw]on't need \d{2} apples''' +lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' + + +################################################################################ +## Integer + +# Integers are whole numbers. Positive numbers may be prefixed with a plus sign. +# Negative numbers are prefixed with a minus sign. + +[integer] + +key1 = +99 +key2 = 42 +key3 = 0 +key4 = -17 + +[integer.underscores] + +# For large numbers, you may use underscores to enhance readability. Each +# underscore must be surrounded by at least one digit. +key1 = 1_000 +key2 = 5_349_221 +key3 = 1_2_3_4_5 # valid but inadvisable + + +################################################################################ +## Float + +# A float consists of an integer part (which may be prefixed with a plus or +# minus sign) followed by a fractional part and/or an exponent part. + +[float.fractional] + +key1 = +1.0 +key2 = 3.1415 +key3 = -0.01 + +[float.exponent] + +key1 = 5e+22 +key2 = 1e6 +key3 = -2E-2 + +[float.both] + +key = 6.626e-34 + +[float.underscores] + +key1 = 9_224_617.445_991_228_313 +key2 = 1e1_00 + + +################################################################################ +## Boolean + +# Booleans are just the tokens you're used to. Always lowercase. + +[boolean] + +True = true +False = false + + +################################################################################ +## Datetime + +# Datetimes are RFC 3339 dates. + +[datetime] + +key1 = 1979-05-27T07:32:00Z +key2 = 1979-05-27T00:32:00-07:00 +key3 = 1979-05-27T00:32:00.999999-07:00 + + +################################################################################ +## Array + +# Arrays are square brackets with other primitives inside. Whitespace is +# ignored. Elements are separated by commas. Data types may not be mixed. + +[array] + +key1 = [ 1, 2, 3 ] +key2 = [ "red", "yellow", "green" ] +key3 = [ [ 1, 2 ], [3, 4, 5] ] +#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok + +# Arrays can also be multiline. So in addition to ignoring whitespace, arrays +# also ignore newlines between the brackets. Terminating commas are ok before +# the closing bracket. + +key5 = [ + 1, 2, 3 +] +key6 = [ + 1, + 2, # this is ok +] + + +################################################################################ +## Array of Tables + +# These can be expressed by using a table name in double brackets. Each table +# with the same double bracketed name will be an element in the array. The +# tables are inserted in the order encountered. + +[[products]] + +name = "Hammer" +sku = 738594937 + +[[products]] + +[[products]] + +name = "Nail" +sku = 284758393 +color = "gray" + + +# You can create nested arrays of tables as well. + +[[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + +[[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" diff --git a/vendor/github.com/pelletier/go-toml/benchmark.yml b/vendor/github.com/pelletier/go-toml/benchmark.yml new file mode 100644 index 0000000000..0bd19f08a6 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.yml @@ -0,0 +1,121 @@ +--- +array: + key1: + - 1 + - 2 + - 3 + key2: + - red + - yellow + - green + key3: + - - 1 + - 2 + - - 3 + - 4 + - 5 + key4: + - - 1 + - 2 + - - a + - b + - c + key5: + - 1 + - 2 + - 3 + key6: + - 1 + - 2 +boolean: + 'False': false + 'True': true +datetime: + key1: '1979-05-27T07:32:00Z' + key2: '1979-05-27T00:32:00-07:00' + key3: '1979-05-27T00:32:00.999999-07:00' +float: + both: + key: 6.626e-34 + exponent: + key1: 5.0e+22 + key2: 1000000 + key3: -0.02 + fractional: + key1: 1 + key2: 3.1415 + key3: -0.01 + underscores: + key1: 9224617.445991227 + key2: 1.0e+100 +fruit: +- name: apple + physical: + color: red + shape: round + variety: + - name: red delicious + - name: granny smith +- name: banana + variety: + - name: plantain +integer: + key1: 99 + key2: 42 + key3: 0 + key4: -17 + underscores: + key1: 1000 + key2: 5349221 + key3: 12345 +products: +- name: Hammer + sku: 738594937 +- {} +- color: gray + name: Nail + sku: 284758393 +string: + basic: + basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + literal: + multiline: + lines: | + The first newline is + trimmed in raw strings. + All other whitespace + is preserved. + regex2: I [dw]on't need \d{2} apples + quoted: Tom "Dubs" Preston-Werner + regex: "<\\i\\c*\\s*>" + winpath: C:\Users\nodejs\templates + winpath2: "\\\\ServerX\\admin$\\system32\\" + multiline: + continued: + key1: The quick brown fox jumps over the lazy dog. + key2: The quick brown fox jumps over the lazy dog. + key3: The quick brown fox jumps over the lazy dog. + key1: |- + One + Two + key2: |- + One + Two + key3: |- + One + Two +table: + inline: + name: + first: Tom + last: Preston-Werner + point: + x: 1 + y: 2 + key: value + subtable: + key: another value +x: + y: + z: + w: {} diff --git a/vendor/github.com/pelletier/go-toml/benchmark_test.go b/vendor/github.com/pelletier/go-toml/benchmark_test.go new file mode 100644 index 0000000000..e1f209dfac --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark_test.go @@ -0,0 +1,192 @@ +package toml + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "testing" + "time" + + burntsushi "github.com/BurntSushi/toml" + yaml "gopkg.in/yaml.v2" +) + +type benchmarkDoc struct { + Table struct { + Key string + Subtable struct { + Key string + } + Inline struct { + Name struct { + First string + Last string + } + Point struct { + X int64 + U int64 + } + } + } + String struct { + Basic struct { + Basic string + } + Multiline struct { + Key1 string + Key2 string + Key3 string + Continued struct { + Key1 string + Key2 string + Key3 string + } + } + Literal struct { + Winpath string + Winpath2 string + Quoted string + Regex string + Multiline struct { + Regex2 string + Lines string + } + } + } + Integer struct { + Key1 int64 + Key2 int64 + Key3 int64 + Key4 int64 + Underscores struct { + Key1 int64 + Key2 int64 + Key3 int64 + } + } + Float struct { + Fractional struct { + Key1 float64 + Key2 float64 + Key3 float64 + } + Exponent struct { + Key1 float64 + Key2 float64 + Key3 float64 + } + Both struct { + Key float64 + } + Underscores struct { + Key1 float64 + Key2 float64 + } + } + Boolean struct { + True bool + False bool + } + Datetime struct { + Key1 time.Time + Key2 time.Time + Key3 time.Time + } + Array struct { + Key1 []int64 + Key2 []string + Key3 [][]int64 + // TODO: Key4 not supported by go-toml's Unmarshal + Key5 []int64 + Key6 []int64 + } + Products []struct { + Name string + Sku int64 + Color string + } + Fruit []struct { + Name string + Physical struct { + Color string + Shape string + Variety []struct { + Name string + } + } + } +} + +func BenchmarkParseToml(b *testing.B) { + fileBytes, err := ioutil.ReadFile("benchmark.toml") + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := LoadReader(bytes.NewReader(fileBytes)) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUnmarshalToml(b *testing.B) { + bytes, err := ioutil.ReadFile("benchmark.toml") + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + target := benchmarkDoc{} + err := Unmarshal(bytes, &target) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUnmarshalBurntSushiToml(b *testing.B) { + bytes, err := ioutil.ReadFile("benchmark.toml") + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + target := benchmarkDoc{} + err := burntsushi.Unmarshal(bytes, &target) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUnmarshalJson(b *testing.B) { + bytes, err := ioutil.ReadFile("benchmark.json") + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + target := benchmarkDoc{} + err := json.Unmarshal(bytes, &target) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkUnmarshalYaml(b *testing.B) { + bytes, err := ioutil.ReadFile("benchmark.yml") + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + target := benchmarkDoc{} + err := yaml.Unmarshal(bytes, &target) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/vendor/github.com/pelletier/go-toml/clean.sh b/vendor/github.com/pelletier/go-toml/clean.sh deleted file mode 100755 index 44d49d936d..0000000000 --- a/vendor/github.com/pelletier/go-toml/clean.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# fail out of the script if anything here fails -set -e - -# clear out stuff generated by test.sh -rm -rf src test_program_bin toml-test diff --git a/vendor/github.com/pelletier/go-toml/doc.go b/vendor/github.com/pelletier/go-toml/doc.go index 9156b736fc..d5fd98c021 100644 --- a/vendor/github.com/pelletier/go-toml/doc.go +++ b/vendor/github.com/pelletier/go-toml/doc.go @@ -1,250 +1,23 @@ -// Package toml is a TOML markup language parser. +// Package toml is a TOML parser and manipulation library. // // This version supports the specification as described in // https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md // -// TOML Parsing +// Marshaling // -// TOML data may be parsed in two ways: by file, or by string. +// Go-toml can marshal and unmarshal TOML documents from and to data +// structures. // -// // load TOML data by filename -// tree, err := toml.LoadFile("filename.toml") +// TOML document as a tree // -// // load TOML data stored in a string -// tree, err := toml.Load(stringContainingTomlData) +// Go-toml can operate on a TOML document as a tree. Use one of the Load* +// functions to parse TOML data and obtain a Tree instance, then one of its +// methods to manipulate the tree. // -// Either way, the result is a TomlTree object that can be used to navigate the -// structure and data within the original document. +// JSONPath-like queries // -// -// Getting data from the TomlTree -// -// After parsing TOML data with Load() or LoadFile(), use the Has() and Get() -// methods on the returned TomlTree, to find your way through the document data. -// -// if tree.Has("foo") { -// fmt.Println("foo is:", tree.Get("foo")) -// } -// -// Working with Paths -// -// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set() -// and GetDefault() methods. These are the same kind of key paths used within the -// TOML specification for struct tames. -// -// // looks for a key named 'baz', within struct 'bar', within struct 'foo' -// tree.Has("foo.bar.baz") -// -// // returns the key at this path, if it is there -// tree.Get("foo.bar.baz") -// -// TOML allows keys to contain '.', which can cause this syntax to be problematic -// for some documents. In such cases, use the GetPath(), HasPath(), and SetPath(), -// methods to explicitly define the path. This form is also faster, since -// it avoids having to parse the passed key for '.' delimiters. -// -// // looks for a key named 'baz', within struct 'bar', within struct 'foo' -// tree.HasPath([]string{"foo","bar","baz"}) -// -// // returns the key at this path, if it is there -// tree.GetPath([]string{"foo","bar","baz"}) -// -// Note that this is distinct from the heavyweight query syntax supported by -// TomlTree.Query() and the Query() struct (see below). -// -// Position Support -// -// Each element within the TomlTree is stored with position metadata, which is -// invaluable for providing semantic feedback to a user. This helps in -// situations where the TOML file parses correctly, but contains data that is -// not correct for the application. In such cases, an error message can be -// generated that indicates the problem line and column number in the source -// TOML document. -// -// // load TOML data -// tree, _ := toml.Load("filename.toml") -// -// // get an entry and report an error if it's the wrong type -// element := tree.Get("foo") -// if value, ok := element.(int64); !ok { -// return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo")) -// } -// -// // report an error if an expected element is missing -// if !tree.Has("bar") { -// return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition("")) -// } -// -// Query Support -// -// The TOML query path implementation is based loosely on the JSONPath specification: -// http://goessner.net/articles/JsonPath/ -// -// The idea behind a query path is to allow quick access to any element, or set -// of elements within TOML document, with a single expression. -// -// result, err := tree.Query("$.foo.bar.baz") -// -// This is roughly equivalent to: -// -// next := tree.Get("foo") -// if next != nil { -// next = next.Get("bar") -// if next != nil { -// next = next.Get("baz") -// } -// } -// result := next -// -// err is nil if any parsing exception occurs. -// -// If no node in the tree matches the query, result will simply contain an empty list of -// items. -// -// As illustrated above, the query path is much more efficient, especially since -// the structure of the TOML file can vary. Rather than making assumptions about -// a document's structure, a query allows the programmer to make structured -// requests into the document, and get zero or more values as a result. -// -// The syntax of a query begins with a root token, followed by any number -// sub-expressions: -// -// $ -// Root of the TOML tree. This must always come first. -// .name -// Selects child of this node, where 'name' is a TOML key -// name. -// ['name'] -// Selects child of this node, where 'name' is a string -// containing a TOML key name. -// [index] -// Selcts child array element at 'index'. -// ..expr -// Recursively selects all children, filtered by an a union, -// index, or slice expression. -// ..* -// Recursive selection of all nodes at this point in the -// tree. -// .* -// Selects all children of the current node. -// [expr,expr] -// Union operator - a logical 'or' grouping of two or more -// sub-expressions: index, key name, or filter. -// [start:end:step] -// Slice operator - selects array elements from start to -// end-1, at the given step. All three arguments are -// optional. -// [?(filter)] -// Named filter expression - the function 'filter' is -// used to filter children at this node. -// -// Query Indexes And Slices -// -// Index expressions perform no bounds checking, and will contribute no -// values to the result set if the provided index or index range is invalid. -// Negative indexes represent values from the end of the array, counting backwards. -// -// // select the last index of the array named 'foo' -// tree.Query("$.foo[-1]") -// -// Slice expressions are supported, by using ':' to separate a start/end index pair. -// -// // select up to the first five elements in the array -// tree.Query("$.foo[0:5]") -// -// Slice expressions also allow negative indexes for the start and stop -// arguments. -// -// // select all array elements. -// tree.Query("$.foo[0:-1]") -// -// Slice expressions may have an optional stride/step parameter: -// -// // select every other element -// tree.Query("$.foo[0:-1:2]") -// -// Slice start and end parameters are also optional: -// -// // these are all equivalent and select all the values in the array -// tree.Query("$.foo[:]") -// tree.Query("$.foo[0:]") -// tree.Query("$.foo[:-1]") -// tree.Query("$.foo[0:-1:]") -// tree.Query("$.foo[::1]") -// tree.Query("$.foo[0::1]") -// tree.Query("$.foo[:-1:1]") -// tree.Query("$.foo[0:-1:1]") -// -// Query Filters -// -// Query filters are used within a Union [,] or single Filter [] expression. -// A filter only allows nodes that qualify through to the next expression, -// and/or into the result set. -// -// // returns children of foo that are permitted by the 'bar' filter. -// tree.Query("$.foo[?(bar)]") -// -// There are several filters provided with the library: -// -// tree -// Allows nodes of type TomlTree. -// int -// Allows nodes of type int64. -// float -// Allows nodes of type float64. -// string -// Allows nodes of type string. -// time -// Allows nodes of type time.Time. -// bool -// Allows nodes of type bool. -// -// Query Results -// -// An executed query returns a QueryResult object. This contains the nodes -// in the TOML tree that qualify the query expression. Position information -// is also available for each value in the set. -// -// // display the results of a query -// results := tree.Query("$.foo.bar.baz") -// for idx, value := results.Values() { -// fmt.Println("%v: %v", results.Positions()[idx], value) -// } -// -// Compiled Queries -// -// Queries may be executed directly on a TomlTree object, or compiled ahead -// of time and executed discretely. The former is more convienent, but has the -// penalty of having to recompile the query expression each time. -// -// // basic query -// results := tree.Query("$.foo.bar.baz") -// -// // compiled query -// query := toml.CompileQuery("$.foo.bar.baz") -// results := query.Execute(tree) -// -// // run the compiled query again on a different tree -// moreResults := query.Execute(anotherTree) -// -// User Defined Query Filters -// -// Filter expressions may also be user defined by using the SetFilter() -// function on the Query object. The function must return true/false, which -// signifies if the passed node is kept or discarded, respectively. -// -// // create a query that references a user-defined filter -// query, _ := CompileQuery("$[?(bazOnly)]") -// -// // define the filter, and assign it to the query -// query.SetFilter("bazOnly", func(node interface{}) bool{ -// if tree, ok := node.(*TomlTree); ok { -// return tree.Has("baz") -// } -// return false // reject all other node types -// }) -// -// // run the query -// query.Execute(tree) +// The package github.com/pelletier/go-toml/query implements a system +// similar to JSONPath to quickly retrieve elements of a TOML document using a +// single expression. See the package documentation for more information. // package toml diff --git a/vendor/github.com/pelletier/go-toml/doc_test.go b/vendor/github.com/pelletier/go-toml/doc_test.go index 69452415a2..3b8171b228 100644 --- a/vendor/github.com/pelletier/go-toml/doc_test.go +++ b/vendor/github.com/pelletier/go-toml/doc_test.go @@ -1,59 +1,16 @@ // code examples for godoc -package toml +package toml_test import ( "fmt" -) - -func ExampleNodeFilterFn_filterExample() { - tree, _ := Load(` - [struct_one] - foo = "foo" - bar = "bar" - - [struct_two] - baz = "baz" - gorf = "gorf" - `) - - // create a query that references a user-defined-filter - query, _ := CompileQuery("$[?(bazOnly)]") - - // define the filter, and assign it to the query - query.SetFilter("bazOnly", func(node interface{}) bool { - if tree, ok := node.(*TomlTree); ok { - return tree.Has("baz") - } - return false // reject all other node types - }) - - // results contain only the 'struct_two' TomlTree - query.Execute(tree) -} - -func ExampleQuery_queryExample() { - config, _ := Load(` - [[book]] - title = "The Stand" - author = "Stephen King" - [[book]] - title = "For Whom the Bell Tolls" - author = "Ernest Hemmingway" - [[book]] - title = "Neuromancer" - author = "William Gibson" - `) + "log" - // find and print all the authors in the document - authors, _ := config.Query("$.book.author") - for _, name := range authors.Values() { - fmt.Println(name) - } -} + toml "github.com/pelletier/go-toml" +) -func Example_comprehensiveExample() { - config, err := LoadFile("config.toml") +func Example_tree() { + config, err := toml.LoadFile("config.toml") if err != nil { fmt.Println("Error ", err.Error()) @@ -63,19 +20,86 @@ func Example_comprehensiveExample() { password := config.Get("postgres.password").(string) // or using an intermediate object - configTree := config.Get("postgres").(*TomlTree) + configTree := config.Get("postgres").(*toml.Tree) user = configTree.Get("user").(string) password = configTree.Get("password").(string) - fmt.Println("User is ", user, ". Password is ", password) + fmt.Println("User is", user, " and password is", password) // show where elements are in the file fmt.Printf("User position: %v\n", configTree.GetPosition("user")) fmt.Printf("Password position: %v\n", configTree.GetPosition("password")) + } +} - // use a query to gather elements without walking the tree - results, _ := config.Query("$..[user,password]") - for ii, item := range results.Values() { - fmt.Printf("Query result %d: %v\n", ii, item) - } +func Example_unmarshal() { + type Employer struct { + Name string + Phone string + } + type Person struct { + Name string + Age int64 + Employer Employer } + + document := []byte(` + name = "John" + age = 30 + [employer] + name = "Company Inc." + phone = "+1 234 567 89012" + `) + + person := Person{} + toml.Unmarshal(document, &person) + fmt.Println(person.Name, "is", person.Age, "and works at", person.Employer.Name) + // Output: + // John is 30 and works at Company Inc. +} + +func ExampleMarshal() { + type Postgres struct { + User string `toml:"user"` + Password string `toml:"password"` + Database string `toml:"db" commented:"true" comment:"not used anymore"` + } + type Config struct { + Postgres Postgres `toml:"postgres" comment:"Postgres configuration"` + } + + config := Config{Postgres{User: "pelletier", Password: "mypassword", Database: "old_database"}} + b, err := toml.Marshal(config) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(b)) + // Output: + // # Postgres configuration + // [postgres] + // + // # not used anymore + // # db = "old_database" + // password = "mypassword" + // user = "pelletier" +} + +func ExampleUnmarshal() { + type Postgres struct { + User string + Password string + } + type Config struct { + Postgres Postgres + } + + doc := []byte(` + [postgres] + user = "pelletier" + password = "mypassword"`) + + config := Config{} + toml.Unmarshal(doc, &config) + fmt.Println("user=", config.Postgres.User) + // Output: + // user= pelletier } diff --git a/vendor/github.com/pelletier/go-toml/fuzz.go b/vendor/github.com/pelletier/go-toml/fuzz.go new file mode 100644 index 0000000000..14570c8d35 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.go @@ -0,0 +1,31 @@ +// +build gofuzz + +package toml + +func Fuzz(data []byte) int { + tree, err := LoadBytes(data) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + str, err := tree.ToTomlString() + if err != nil { + if str != "" { + panic(`str must be "" if there is an error`) + } + panic(err) + } + + tree, err = Load(str) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + return 1 +} diff --git a/vendor/github.com/pelletier/go-toml/fuzz.sh b/vendor/github.com/pelletier/go-toml/fuzz.sh new file mode 100755 index 0000000000..3204b4c446 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.sh @@ -0,0 +1,15 @@ +#! /bin/sh +set -eu + +go get github.com/dvyukov/go-fuzz/go-fuzz +go get github.com/dvyukov/go-fuzz/go-fuzz-build + +if [ ! -e toml-fuzz.zip ]; then + go-fuzz-build github.com/pelletier/go-toml +fi + +rm -fr fuzz +mkdir -p fuzz/corpus +cp *.toml fuzz/corpus + +go-fuzz -bin=toml-fuzz.zip -workdir=fuzz diff --git a/vendor/github.com/pelletier/go-toml/keysparsing.go b/vendor/github.com/pelletier/go-toml/keysparsing.go index d62ca5fd1d..9707c68840 100644 --- a/vendor/github.com/pelletier/go-toml/keysparsing.go +++ b/vendor/github.com/pelletier/go-toml/keysparsing.go @@ -6,15 +6,37 @@ import ( "bytes" "errors" "fmt" + "strconv" "unicode" ) +var escapeSequenceMap = map[rune]rune{ + 'b': '\b', + 't': '\t', + 'n': '\n', + 'f': '\f', + 'r': '\r', + '"': '"', + '\\': '\\', +} + +type parseKeyState int + +const ( + BARE parseKeyState = iota + BASIC + LITERAL + ESC + UNICODE_4 + UNICODE_8 +) + func parseKey(key string) ([]string, error) { groups := []string{} var buffer bytes.Buffer - inQuotes := false + var hex bytes.Buffer + state := BARE wasInQuotes := false - escapeNext := false ignoreSpace := true expectDot := false @@ -25,25 +47,67 @@ func parseKey(key string) ([]string, error) { } ignoreSpace = false } - if escapeNext { - buffer.WriteRune(char) - escapeNext = false + + if state == ESC { + if char == 'u' { + state = UNICODE_4 + hex.Reset() + } else if char == 'U' { + state = UNICODE_8 + hex.Reset() + } else if newChar, ok := escapeSequenceMap[char]; ok { + buffer.WriteRune(newChar) + state = BASIC + } else { + return nil, fmt.Errorf(`invalid escape sequence \%c`, char) + } + continue + } + + if state == UNICODE_4 || state == UNICODE_8 { + if isHexDigit(char) { + hex.WriteRune(char) + } + if (state == UNICODE_4 && hex.Len() == 4) || (state == UNICODE_8 && hex.Len() == 8) { + if value, err := strconv.ParseInt(hex.String(), 16, 32); err == nil { + buffer.WriteRune(rune(value)) + } else { + return nil, err + } + state = BASIC + } continue } + switch char { case '\\': - escapeNext = true - continue + if state == BASIC { + state = ESC + } else if state == LITERAL { + buffer.WriteRune(char) + } + case '\'': + if state == BARE { + state = LITERAL + } else if state == LITERAL { + groups = append(groups, buffer.String()) + buffer.Reset() + wasInQuotes = true + state = BARE + } + expectDot = false case '"': - if inQuotes { + if state == BARE { + state = BASIC + } else if state == BASIC { groups = append(groups, buffer.String()) buffer.Reset() + state = BARE wasInQuotes = true } - inQuotes = !inQuotes expectDot = false case '.': - if inQuotes { + if state != BARE { buffer.WriteRune(char) } else { if !wasInQuotes { @@ -58,28 +122,31 @@ func parseKey(key string) ([]string, error) { wasInQuotes = false } case ' ': - if inQuotes { + if state == BASIC { buffer.WriteRune(char) } else { expectDot = true } default: - if !inQuotes && !isValidBareChar(char) { - return nil, fmt.Errorf("invalid bare character: %c", char) - } - if !inQuotes && expectDot { - return nil, errors.New("what?") + if state == BARE { + if !isValidBareChar(char) { + return nil, fmt.Errorf("invalid bare character: %c", char) + } else if expectDot { + return nil, errors.New("what?") + } } buffer.WriteRune(char) expectDot = false } } - if inQuotes { - return nil, errors.New("mismatched quotes") - } - if escapeNext { + + // state must be BARE at the end + if state == ESC { return nil, errors.New("unfinished escape sequence") + } else if state != BARE { + return nil, errors.New("mismatched quotes") } + if buffer.Len() > 0 { groups = append(groups, buffer.String()) } diff --git a/vendor/github.com/pelletier/go-toml/keysparsing_test.go b/vendor/github.com/pelletier/go-toml/keysparsing_test.go index 1a9ecccaa9..7aa4cd64a6 100644 --- a/vendor/github.com/pelletier/go-toml/keysparsing_test.go +++ b/vendor/github.com/pelletier/go-toml/keysparsing_test.go @@ -22,7 +22,10 @@ func testResult(t *testing.T, key string, expected []string) { } func testError(t *testing.T, key string, expectedError string) { - _, err := parseKey(key) + res, err := parseKey(key) + if err == nil { + t.Fatalf("Expected error, but succesfully parsed key %s", res) + } if fmt.Sprintf("%s", err) != expectedError { t.Fatalf("Expected error \"%s\", but got \"%s\".", expectedError, err) } @@ -47,6 +50,17 @@ func TestBaseKeyPound(t *testing.T) { func TestQuotedKeys(t *testing.T) { testResult(t, `hello."foo".bar`, []string{"hello", "foo", "bar"}) testResult(t, `"hello!"`, []string{"hello!"}) + testResult(t, `"hello\tworld"`, []string{"hello\tworld"}) + testResult(t, `"\U0001F914"`, []string{"\U0001F914"}) + testResult(t, `"\u2764"`, []string{"\u2764"}) + + testResult(t, `hello.'foo'.bar`, []string{"hello", "foo", "bar"}) + testResult(t, `'hello!'`, []string{"hello!"}) + testResult(t, `'hello\tworld'`, []string{`hello\tworld`}) + + testError(t, `"\w"`, `invalid escape sequence \w`) + testError(t, `"\`, `unfinished escape sequence`) + testError(t, `"\t`, `mismatched quotes`) } func TestEmptyKey(t *testing.T) { diff --git a/vendor/github.com/pelletier/go-toml/lexer.go b/vendor/github.com/pelletier/go-toml/lexer.go index 104f3b1f46..1b6647d66f 100644 --- a/vendor/github.com/pelletier/go-toml/lexer.go +++ b/vendor/github.com/pelletier/go-toml/lexer.go @@ -6,14 +6,12 @@ package toml import ( + "bytes" "errors" "fmt" - "io" "regexp" "strconv" "strings" - - "github.com/pelletier/go-buffruneio" ) var dateRegexp *regexp.Regexp @@ -23,29 +21,29 @@ type tomlLexStateFn func() tomlLexStateFn // Define lexer type tomlLexer struct { - input *buffruneio.Reader // Textual source - buffer []rune // Runes composing the current token - tokens chan token - depth int - line int - col int - endbufferLine int - endbufferCol int + inputIdx int + input []rune // Textual source + currentTokenStart int + currentTokenStop int + tokens []token + depth int + line int + col int + endbufferLine int + endbufferCol int } // Basic read operations on input func (l *tomlLexer) read() rune { - r, _, err := l.input.ReadRune() - if err != nil { - panic(err) - } + r := l.peek() if r == '\n' { l.endbufferLine++ l.endbufferCol = 1 } else { l.endbufferCol++ } + l.inputIdx++ return r } @@ -53,13 +51,13 @@ func (l *tomlLexer) next() rune { r := l.read() if r != eof { - l.buffer = append(l.buffer, r) + l.currentTokenStop++ } return r } func (l *tomlLexer) ignore() { - l.buffer = make([]rune, 0) + l.currentTokenStart = l.currentTokenStop l.line = l.endbufferLine l.col = l.endbufferCol } @@ -76,49 +74,46 @@ func (l *tomlLexer) fastForward(n int) { } func (l *tomlLexer) emitWithValue(t tokenType, value string) { - l.tokens <- token{ + l.tokens = append(l.tokens, token{ Position: Position{l.line, l.col}, typ: t, val: value, - } + }) l.ignore() } func (l *tomlLexer) emit(t tokenType) { - l.emitWithValue(t, string(l.buffer)) + l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop])) } func (l *tomlLexer) peek() rune { - r, _, err := l.input.ReadRune() - if err != nil { - panic(err) + if l.inputIdx >= len(l.input) { + return eof } - l.input.UnreadRune() - return r + return l.input[l.inputIdx] } -func (l *tomlLexer) follow(next string) bool { - for _, expectedRune := range next { - r, _, err := l.input.ReadRune() - defer l.input.UnreadRune() - if err != nil { - panic(err) - } - if expectedRune != r { - return false - } +func (l *tomlLexer) peekString(size int) string { + maxIdx := len(l.input) + upperIdx := l.inputIdx + size // FIXME: potential overflow + if upperIdx > maxIdx { + upperIdx = maxIdx } - return true + return string(l.input[l.inputIdx:upperIdx]) +} + +func (l *tomlLexer) follow(next string) bool { + return next == l.peekString(len(next)) } // Error management func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn { - l.tokens <- token{ + l.tokens = append(l.tokens, token{ Position: Position{l.line, l.col}, typ: tokenError, val: fmt.Sprintf(format, args...), - } + }) return nil } @@ -219,7 +214,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { break } - possibleDate := string(l.input.PeekRunes(35)) + possibleDate := l.peekString(35) dateMatch := dateRegexp.FindString(possibleDate) if dateMatch != "" { l.fastForward(len(dateMatch)) @@ -536,7 +531,7 @@ func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.currentTokenStop > l.currentTokenStart { l.emit(tokenKeyGroupArray) } l.next() @@ -559,7 +554,7 @@ func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.currentTokenStop > l.currentTokenStart { l.emit(tokenKeyGroup) } l.next() @@ -634,7 +629,6 @@ func (l *tomlLexer) run() { for state := l.lexVoid; state != nil; { state = state() } - close(l.tokens) } func init() { @@ -642,16 +636,16 @@ func init() { } // Entry point -func lexToml(input io.Reader) chan token { - bufferedInput := buffruneio.NewReader(input) +func lexToml(inputBytes []byte) []token { + runes := bytes.Runes(inputBytes) l := &tomlLexer{ - input: bufferedInput, - tokens: make(chan token), + input: runes, + tokens: make([]token, 0, 256), line: 1, col: 1, endbufferLine: 1, endbufferCol: 1, } - go l.run() + l.run() return l.tokens } diff --git a/vendor/github.com/pelletier/go-toml/lexer_test.go b/vendor/github.com/pelletier/go-toml/lexer_test.go index 6b324ea0e0..313b83c5d1 100644 --- a/vendor/github.com/pelletier/go-toml/lexer_test.go +++ b/vendor/github.com/pelletier/go-toml/lexer_test.go @@ -1,37 +1,14 @@ package toml import ( - "strings" + "reflect" "testing" ) func testFlow(t *testing.T, input string, expectedFlow []token) { - ch := lexToml(strings.NewReader(input)) - for _, expected := range expectedFlow { - token := <-ch - if token != expected { - t.Log("While testing: ", input) - t.Log("compared (got)", token, "to (expected)", expected) - t.Log("\tvalue:", token.val, "<->", expected.val) - t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val)) - t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String()) - t.Log("\tline:", token.Line, "<->", expected.Line) - t.Log("\tcolumn:", token.Col, "<->", expected.Col) - t.Log("compared", token, "to", expected) - t.FailNow() - } - } - - tok, ok := <-ch - if ok { - t.Log("channel is not closed!") - t.Log(len(ch)+1, "tokens remaining:") - - t.Log("token ->", tok) - for token := range ch { - t.Log("token ->", token) - } - t.FailNow() + tokens := lexToml([]byte(input)) + if !reflect.DeepEqual(tokens, expectedFlow) { + t.Fatal("Different flows. Expected\n", expectedFlow, "\nGot:\n", tokens) } } @@ -748,3 +725,26 @@ func TestLexUnknownRvalue(t *testing.T) { {Position{1, 5}, tokenError, `no value can start with \`}, }) } + +func BenchmarkLexer(b *testing.B) { + sample := `title = "Hugo: A Fast and Flexible Website Generator" +baseurl = "http://gohugo.io/" +MetaDataFormat = "yaml" +pluralizeListTitles = false + +[params] + description = "Documentation of Hugo, a fast and flexible static site generator built with love by spf13, bep and friends in Go" + author = "Steve Francia (spf13) and friends" + release = "0.22-DEV" + +[[menu.main]] + name = "Download Hugo" + pre = "" + url = "https://github.com/spf13/hugo/releases" + weight = -200 +` + b.ResetTimer() + for i := 0; i < b.N; i++ { + lexToml([]byte(sample)) + } +} diff --git a/vendor/github.com/pelletier/go-toml/marshal.go b/vendor/github.com/pelletier/go-toml/marshal.go index a1d7010468..e6281e9d23 100644 --- a/vendor/github.com/pelletier/go-toml/marshal.go +++ b/vendor/github.com/pelletier/go-toml/marshal.go @@ -1,41 +1,37 @@ package toml import ( + "bytes" "errors" "fmt" + "io" "reflect" + "strconv" "strings" "time" ) -/* -TomlTree structural types and corresponding marshal types -------------------------------------------------------------------------------- -*TomlTree (*)struct, (*)map[string]interface{} -[]*TomlTree (*)[](*)struct, (*)[](*)map[string]interface{} -[]interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) -interface{} (*)primitive - -TomlTree primitive types and corresponding marshal types ------------------------------------------------------------ -uint64 uint, uint8-uint64, pointers to same -int64 int, int8-uint64, pointers to same -float64 float32, float64, pointers to same -string string, pointers to same -bool bool, pointers to same -time.Time time.Time{}, pointers to same -*/ - type tomlOpts struct { name string + comment string + commented bool include bool omitempty bool } +type encOpts struct { + quoteMapKeys bool + arraysOneElementPerLine bool +} + +var encOptsDefaults = encOpts{ + quoteMapKeys: false, +} + var timeType = reflect.TypeOf(time.Time{}) var marshalerType = reflect.TypeOf(new(Marshaler)).Elem() -// Check if the given marshall type maps to a TomlTree primitive +// Check if the given marshall type maps to a Tree primitive func isPrimitive(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Ptr: @@ -57,7 +53,7 @@ func isPrimitive(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a TomlTree slice +// Check if the given marshall type maps to a Tree slice func isTreeSlice(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Slice: @@ -67,7 +63,7 @@ func isTreeSlice(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a non-TomlTree slice +// Check if the given marshall type maps to a non-Tree slice func isOtherSlice(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Ptr: @@ -79,7 +75,7 @@ func isOtherSlice(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a TomlTree +// Check if the given marshall type maps to a Tree func isTree(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Map: @@ -111,11 +107,94 @@ encoder, except that there is no concept of a Marshaler interface or MarshalTOML function for sub-structs, and currently only definite types can be marshaled (i.e. no `interface{}`). +The following struct annotations are supported: + + toml:"Field" Overrides the field's name to output. + omitempty When set, empty values and groups are not emitted. + comment:"comment" Emits a # comment on the same line. This supports new lines. + commented:"true" Emits the value as commented. + Note that pointers are automatically assigned the "omitempty" option, as TOML -explicity does not handle null values (saying instead the label should be +explicitly does not handle null values (saying instead the label should be dropped). + +Tree structural types and corresponding marshal types: + + *Tree (*)struct, (*)map[string]interface{} + []*Tree (*)[](*)struct, (*)[](*)map[string]interface{} + []interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) + interface{} (*)primitive + +Tree primitive types and corresponding marshal types: + + uint64 uint, uint8-uint64, pointers to same + int64 int, int8-uint64, pointers to same + float64 float32, float64, pointers to same + string string, pointers to same + bool bool, pointers to same + time.Time time.Time{}, pointers to same */ func Marshal(v interface{}) ([]byte, error) { + return NewEncoder(nil).marshal(v) +} + +// Encoder writes TOML values to an output stream. +type Encoder struct { + w io.Writer + encOpts +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: w, + encOpts: encOptsDefaults, + } +} + +// Encode writes the TOML encoding of v to the stream. +// +// See the documentation for Marshal for details. +func (e *Encoder) Encode(v interface{}) error { + b, err := e.marshal(v) + if err != nil { + return err + } + if _, err := e.w.Write(b); err != nil { + return err + } + return nil +} + +// QuoteMapKeys sets up the encoder to encode +// maps with string type keys with quoted TOML keys. +// +// This relieves the character limitations on map keys. +func (e *Encoder) QuoteMapKeys(v bool) *Encoder { + e.quoteMapKeys = v + return e +} + +// ArraysWithOneElementPerLine sets up the encoder to encode arrays +// with more than one element on multiple lines instead of one. +// +// For example: +// +// A = [1,2,3] +// +// Becomes +// +// A = [ +// 1, +// 2, +// 3 +// ] +func (e *Encoder) ArraysWithOneElementPerLine(v bool) *Encoder { + e.arraysOneElementPerLine = v + return e +} + +func (e *Encoder) marshal(v interface{}) ([]byte, error) { mtype := reflect.TypeOf(v) if mtype.Kind() != reflect.Struct { return []byte{}, errors.New("Only a struct can be marshaled to TOML") @@ -124,51 +203,62 @@ func Marshal(v interface{}) ([]byte, error) { if isCustomMarshaler(mtype) { return callCustomMarshaler(sval) } - t, err := valueToTree(mtype, sval) + t, err := e.valueToTree(mtype, sval) if err != nil { return []byte{}, err } - s, err := t.ToTomlString() - return []byte(s), err + + var buf bytes.Buffer + _, err = t.writeTo(&buf, "", "", 0, e.arraysOneElementPerLine) + + return buf.Bytes(), err } // Convert given marshal struct or map value to toml tree -func valueToTree(mtype reflect.Type, mval reflect.Value) (*TomlTree, error) { +func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) { if mtype.Kind() == reflect.Ptr { - return valueToTree(mtype.Elem(), mval.Elem()) + return e.valueToTree(mtype.Elem(), mval.Elem()) } - tval := newTomlTree() + tval := newTree() switch mtype.Kind() { case reflect.Struct: for i := 0; i < mtype.NumField(); i++ { mtypef, mvalf := mtype.Field(i), mval.Field(i) opts := tomlOptions(mtypef) if opts.include && (!opts.omitempty || !isZero(mvalf)) { - val, err := valueToToml(mtypef.Type, mvalf) + val, err := e.valueToToml(mtypef.Type, mvalf) if err != nil { return nil, err } - tval.Set(opts.name, val) + tval.Set(opts.name, opts.comment, opts.commented, val) } } case reflect.Map: for _, key := range mval.MapKeys() { mvalf := mval.MapIndex(key) - val, err := valueToToml(mtype.Elem(), mvalf) + val, err := e.valueToToml(mtype.Elem(), mvalf) if err != nil { return nil, err } - tval.Set(key.String(), val) + if e.quoteMapKeys { + keyStr, err := tomlValueStringRepresentation(key.String(), "", e.arraysOneElementPerLine) + if err != nil { + return nil, err + } + tval.SetPath([]string{keyStr}, "", false, val) + } else { + tval.Set(key.String(), "", false, val) + } } } return tval, nil } // Convert given marshal slice to slice of Toml trees -func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*TomlTree, error) { - tval := make([]*TomlTree, mval.Len(), mval.Len()) +func (e *Encoder) valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) { + tval := make([]*Tree, mval.Len(), mval.Len()) for i := 0; i < mval.Len(); i++ { - val, err := valueToTree(mtype.Elem(), mval.Index(i)) + val, err := e.valueToTree(mtype.Elem(), mval.Index(i)) if err != nil { return nil, err } @@ -178,10 +268,10 @@ func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*TomlTree, erro } // Convert given marshal slice to slice of toml values -func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { +func (e *Encoder) valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { tval := make([]interface{}, mval.Len(), mval.Len()) for i := 0; i < mval.Len(); i++ { - val, err := valueToToml(mtype.Elem(), mval.Index(i)) + val, err := e.valueToToml(mtype.Elem(), mval.Index(i)) if err != nil { return nil, err } @@ -191,19 +281,19 @@ func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, err } // Convert given marshal value to toml value -func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { +func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { if mtype.Kind() == reflect.Ptr { - return valueToToml(mtype.Elem(), mval.Elem()) + return e.valueToToml(mtype.Elem(), mval.Elem()) } switch { case isCustomMarshaler(mtype): return callCustomMarshaler(mval) case isTree(mtype): - return valueToTree(mtype, mval) + return e.valueToTree(mtype, mval) case isTreeSlice(mtype): - return valueToTreeSlice(mtype, mval) + return e.valueToTreeSlice(mtype, mval) case isOtherSlice(mtype): - return valueToOtherSlice(mtype, mval) + return e.valueToOtherSlice(mtype, mval) default: switch mtype.Kind() { case reflect.Bool: @@ -224,25 +314,76 @@ func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { } } -/* -Unmarshal parses the TOML-encoded data and stores the result in the value -pointed to by v. Behavior is similar to the Go json encoder, except that there -is no concept of an Unmarshaler interface or UnmarshalTOML function for -sub-structs, and currently only definite types can be unmarshaled to (i.e. no -`interface{}`). -*/ +// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v. +// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for +// sub-structs, and only definite types can be unmarshaled. +func (t *Tree) Unmarshal(v interface{}) error { + d := Decoder{tval: t} + return d.unmarshal(v) +} + +// Marshal returns the TOML encoding of Tree. +// See Marshal() documentation for types mapping table. +func (t *Tree) Marshal() ([]byte, error) { + var buf bytes.Buffer + err := NewEncoder(&buf).Encode(t) + return buf.Bytes(), err +} + +// Unmarshal parses the TOML-encoded data and stores the result in the value +// pointed to by v. Behavior is similar to the Go json encoder, except that there +// is no concept of an Unmarshaler interface or UnmarshalTOML function for +// sub-structs, and currently only definite types can be unmarshaled to (i.e. no +// `interface{}`). +// +// The following struct annotations are supported: +// +// toml:"Field" Overrides the field's name to map to. +// +// See Marshal() documentation for types mapping table. func Unmarshal(data []byte, v interface{}) error { - mtype := reflect.TypeOf(v) - if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct { - return errors.New("Only a pointer to struct can be unmarshaled from TOML") + t, err := LoadReader(bytes.NewReader(data)) + if err != nil { + return err } + return t.Unmarshal(v) +} + +// Decoder reads and decodes TOML values from an input stream. +type Decoder struct { + r io.Reader + tval *Tree + encOpts +} - t, err := Load(string(data)) +// NewDecoder returns a new decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + r: r, + encOpts: encOptsDefaults, + } +} + +// Decode reads a TOML-encoded value from it's input +// and unmarshals it in the value pointed at by v. +// +// See the documentation for Marshal for details. +func (d *Decoder) Decode(v interface{}) error { + var err error + d.tval, err = LoadReader(d.r) if err != nil { return err } + return d.unmarshal(v) +} - sval, err := valueFromTree(mtype.Elem(), t) +func (d *Decoder) unmarshal(v interface{}) error { + mtype := reflect.TypeOf(v) + if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct { + return errors.New("Only a pointer to struct can be unmarshaled from TOML") + } + + sval, err := d.valueFromTree(mtype.Elem(), d.tval) if err != nil { return err } @@ -251,9 +392,9 @@ func Unmarshal(data []byte, v interface{}) error { } // Convert toml tree to marshal struct or map, using marshal type -func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { +func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) { if mtype.Kind() == reflect.Ptr { - return unwrapPointer(mtype, tval) + return d.unwrapPointer(mtype, tval) } var mval reflect.Value switch mtype.Kind() { @@ -263,23 +404,29 @@ func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { mtypef := mtype.Field(i) opts := tomlOptions(mtypef) if opts.include { - key := opts.name - exists := tval.Has(key) - if exists { + baseKey := opts.name + keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)} + for _, key := range keysToTry { + exists := tval.Has(key) + if !exists { + continue + } val := tval.Get(key) - mvalf, err := valueFromToml(mtypef.Type, val) + mvalf, err := d.valueFromToml(mtypef.Type, val) if err != nil { return mval, formatError(err, tval.GetPosition(key)) } mval.Field(i).Set(mvalf) + break } } } case reflect.Map: mval = reflect.MakeMap(mtype) for _, key := range tval.Keys() { - val := tval.Get(key) - mvalf, err := valueFromToml(mtype.Elem(), val) + // TODO: path splits key + val := tval.GetPath([]string{key}) + mvalf, err := d.valueFromToml(mtype.Elem(), val) if err != nil { return mval, formatError(err, tval.GetPosition(key)) } @@ -290,10 +437,10 @@ func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { } // Convert toml value to marshal struct/map slice, using marshal type -func valueFromTreeSlice(mtype reflect.Type, tval []*TomlTree) (reflect.Value, error) { +func (d *Decoder) valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) { mval := reflect.MakeSlice(mtype, len(tval), len(tval)) for i := 0; i < len(tval); i++ { - val, err := valueFromTree(mtype.Elem(), tval[i]) + val, err := d.valueFromTree(mtype.Elem(), tval[i]) if err != nil { return mval, err } @@ -303,10 +450,10 @@ func valueFromTreeSlice(mtype reflect.Type, tval []*TomlTree) (reflect.Value, er } // Convert toml value to marshal primitive slice, using marshal type -func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { +func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { mval := reflect.MakeSlice(mtype, len(tval), len(tval)) for i := 0; i < len(tval); i++ { - val, err := valueFromToml(mtype.Elem(), tval[i]) + val, err := d.valueFromToml(mtype.Elem(), tval[i]) if err != nil { return mval, err } @@ -316,17 +463,30 @@ func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, } // Convert toml value to marshal value, using marshal type -func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { +func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { if mtype.Kind() == reflect.Ptr { - return unwrapPointer(mtype, tval) + return d.unwrapPointer(mtype, tval) } - switch { - case isTree(mtype): - return valueFromTree(mtype, tval.(*TomlTree)) - case isTreeSlice(mtype): - return valueFromTreeSlice(mtype, tval.([]*TomlTree)) - case isOtherSlice(mtype): - return valueFromOtherSlice(mtype, tval.([]interface{})) + + switch tval.(type) { + case *Tree: + if isTree(mtype) { + return d.valueFromTree(mtype, tval.(*Tree)) + } else { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a tree", tval, tval) + } + case []*Tree: + if isTreeSlice(mtype) { + return d.valueFromTreeSlice(mtype, tval.([]*Tree)) + } else { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to trees", tval, tval) + } + case []interface{}: + if isOtherSlice(mtype) { + return d.valueFromOtherSlice(mtype, tval.([]interface{})) + } else { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a slice", tval, tval) + } default: switch mtype.Kind() { case reflect.Bool: @@ -420,13 +580,13 @@ func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) } return reflect.ValueOf(val), nil default: - return reflect.ValueOf(nil), fmt.Errorf("Unmarshal can't handle %v(%v)", mtype, mtype.Kind()) + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind()) } } } -func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { - val, err := valueFromToml(mtype.Elem(), tval) +func (d *Decoder) unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + val, err := d.valueFromToml(mtype.Elem(), tval) if err != nil { return reflect.ValueOf(nil), err } @@ -438,7 +598,12 @@ func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) func tomlOptions(vf reflect.StructField) tomlOpts { tag := vf.Tag.Get("toml") parse := strings.Split(tag, ",") - result := tomlOpts{vf.Name, true, false} + var comment string + if c := vf.Tag.Get("comment"); c != "" { + comment = c + } + commented, _ := strconv.ParseBool(vf.Tag.Get("commented")) + result := tomlOpts{name: vf.Name, comment: comment, commented: commented, include: true, omitempty: false} if parse[0] != "" { if parse[0] == "-" && len(parse) == 1 { result.include = false diff --git a/vendor/github.com/pelletier/go-toml/marshal_test.go b/vendor/github.com/pelletier/go-toml/marshal_test.go index 891222e9b1..82268aa7f6 100644 --- a/vendor/github.com/pelletier/go-toml/marshal_test.go +++ b/vendor/github.com/pelletier/go-toml/marshal_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "reflect" + "strings" "testing" "time" ) @@ -177,6 +178,23 @@ func TestDocUnmarshal(t *testing.T) { } } +func TestDocPartialUnmarshal(t *testing.T) { + result := testDocSubs{} + + tree, _ := LoadFile("marshal_test.toml") + subTree := tree.Get("subdoc").(*Tree) + err := subTree.Unmarshal(&result) + expected := docData.Subdocs + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result, expected) { + resStr, _ := json.MarshalIndent(result, "", " ") + expStr, _ := json.MarshalIndent(expected, "", " ") + t.Errorf("Bad partial unmartial: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expStr, resStr) + } +} + type tomlTypeCheckTest struct { name string item interface{} @@ -491,6 +509,14 @@ func TestPointerUnmarshal(t *testing.T) { } } +func TestUnmarshalTypeMismatch(t *testing.T) { + result := pointerMarshalTestStruct{} + err := Unmarshal([]byte("List = 123"), &result) + if !strings.HasPrefix(err.Error(), "(1, 1): Can't convert 123(int64) to []string(slice)") { + t.Errorf("Type mismatch must be reported: got %v", err.Error()) + } +} + type nestedMarshalTestStruct struct { String [][]string //Struct [][]basicMarshalTestSubStruct @@ -581,3 +607,200 @@ func TestNestedCustomMarshaler(t *testing.T) { t.Errorf("Bad nested custom marshaler: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result) } } + +var commentTestToml = []byte(` +# it's a comment on type +[postgres] + # isCommented = "dvalue" + noComment = "cvalue" + + # A comment on AttrB with a + # break line + password = "bvalue" + + # A comment on AttrA + user = "avalue" + + [[postgres.My]] + + # a comment on my on typeC + My = "Foo" + + [[postgres.My]] + + # a comment on my on typeC + My = "Baar" +`) + +func TestMarshalComment(t *testing.T) { + type TypeC struct { + My string `comment:"a comment on my on typeC"` + } + type TypeB struct { + AttrA string `toml:"user" comment:"A comment on AttrA"` + AttrB string `toml:"password" comment:"A comment on AttrB with a\n break line"` + AttrC string `toml:"noComment"` + AttrD string `toml:"isCommented" commented:"true"` + My []TypeC + } + type TypeA struct { + TypeB TypeB `toml:"postgres" comment:"it's a comment on type"` + } + + ta := []TypeC{{My: "Foo"}, {My: "Baar"}} + config := TypeA{TypeB{AttrA: "avalue", AttrB: "bvalue", AttrC: "cvalue", AttrD: "dvalue", My: ta}} + result, err := Marshal(config) + if err != nil { + t.Fatal(err) + } + expected := commentTestToml + if !bytes.Equal(result, expected) { + t.Errorf("Bad marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result) + } +} + +type mapsTestStruct struct { + Simple map[string]string + Paths map[string]string + Other map[string]float64 + X struct { + Y struct { + Z map[string]bool + } + } +} + +var mapsTestData = mapsTestStruct{ + Simple: map[string]string{ + "one plus one": "two", + "next": "three", + }, + Paths: map[string]string{ + "/this/is/a/path": "/this/is/also/a/path", + "/heloo.txt": "/tmp/lololo.txt", + }, + Other: map[string]float64{ + "testing": 3.9999, + }, + X: struct{ Y struct{ Z map[string]bool } }{ + Y: struct{ Z map[string]bool }{ + Z: map[string]bool{ + "is.Nested": true, + }, + }, + }, +} +var mapsTestToml = []byte(` +[Other] + "testing" = 3.9999 + +[Paths] + "/heloo.txt" = "/tmp/lololo.txt" + "/this/is/a/path" = "/this/is/also/a/path" + +[Simple] + "next" = "three" + "one plus one" = "two" + +[X] + + [X.Y] + + [X.Y.Z] + "is.Nested" = true +`) + +func TestEncodeQuotedMapKeys(t *testing.T) { + var buf bytes.Buffer + if err := NewEncoder(&buf).QuoteMapKeys(true).Encode(mapsTestData); err != nil { + t.Fatal(err) + } + result := buf.Bytes() + expected := mapsTestToml + if !bytes.Equal(result, expected) { + t.Errorf("Bad maps marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, result) + } +} + +func TestDecodeQuotedMapKeys(t *testing.T) { + result := mapsTestStruct{} + err := NewDecoder(bytes.NewBuffer(mapsTestToml)).Decode(&result) + expected := mapsTestData + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(result, expected) { + t.Errorf("Bad maps unmarshal: expected %v, got %v", expected, result) + } +} + +type structArrayNoTag struct { + A struct { + B []int64 + C []int64 + } +} + +func TestMarshalArray(t *testing.T) { + expected := []byte(` +[A] + B = [1,2,3] + C = [1] +`) + + m := structArrayNoTag{ + A: struct { + B []int64 + C []int64 + }{ + B: []int64{1, 2, 3}, + C: []int64{1}, + }, + } + + b, err := Marshal(m) + + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(b, expected) { + t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b) + } +} + +func TestMarshalArrayOnePerLine(t *testing.T) { + expected := []byte(` +[A] + B = [ + 1, + 2, + 3 + ] + C = [1] +`) + + m := structArrayNoTag{ + A: struct { + B []int64 + C []int64 + }{ + B: []int64{1, 2, 3}, + C: []int64{1}, + }, + } + + var buf bytes.Buffer + encoder := NewEncoder(&buf).ArraysWithOneElementPerLine(true) + err := encoder.Encode(m) + + if err != nil { + t.Fatal(err) + } + + b := buf.Bytes() + + if !bytes.Equal(b, expected) { + t.Errorf("Bad arrays marshal: expected\n-----\n%s\n-----\ngot\n-----\n%s\n-----\n", expected, b) + } +} diff --git a/vendor/github.com/pelletier/go-toml/match.go b/vendor/github.com/pelletier/go-toml/match.go deleted file mode 100644 index 48b0f2a153..0000000000 --- a/vendor/github.com/pelletier/go-toml/match.go +++ /dev/null @@ -1,234 +0,0 @@ -package toml - -import ( - "fmt" -) - -// support function to set positions for tomlValues -// NOTE: this is done to allow ctx.lastPosition to indicate the start of any -// values returned by the query engines -func tomlValueCheck(node interface{}, ctx *queryContext) interface{} { - switch castNode := node.(type) { - case *tomlValue: - ctx.lastPosition = castNode.position - return castNode.value - case []*TomlTree: - if len(castNode) > 0 { - ctx.lastPosition = castNode[0].position - } - return node - default: - return node - } -} - -// base match -type matchBase struct { - next pathFn -} - -func (f *matchBase) setNext(next pathFn) { - f.next = next -} - -// terminating functor - gathers results -type terminatingFn struct { - // empty -} - -func newTerminatingFn() *terminatingFn { - return &terminatingFn{} -} - -func (f *terminatingFn) setNext(next pathFn) { - // do nothing -} - -func (f *terminatingFn) call(node interface{}, ctx *queryContext) { - switch castNode := node.(type) { - case *TomlTree: - ctx.result.appendResult(node, castNode.position) - case *tomlValue: - ctx.result.appendResult(node, castNode.position) - default: - // use last position for scalars - ctx.result.appendResult(node, ctx.lastPosition) - } -} - -// match single key -type matchKeyFn struct { - matchBase - Name string -} - -func newMatchKeyFn(name string) *matchKeyFn { - return &matchKeyFn{Name: name} -} - -func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { - if array, ok := node.([]*TomlTree); ok { - for _, tree := range array { - item := tree.values[f.Name] - if item != nil { - f.next.call(item, ctx) - } - } - } else if tree, ok := node.(*TomlTree); ok { - item := tree.values[f.Name] - if item != nil { - f.next.call(item, ctx) - } - } -} - -// match single index -type matchIndexFn struct { - matchBase - Idx int -} - -func newMatchIndexFn(idx int) *matchIndexFn { - return &matchIndexFn{Idx: idx} -} - -func (f *matchIndexFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { - if f.Idx < len(arr) && f.Idx >= 0 { - f.next.call(arr[f.Idx], ctx) - } - } -} - -// filter by slicing -type matchSliceFn struct { - matchBase - Start, End, Step int -} - -func newMatchSliceFn(start, end, step int) *matchSliceFn { - return &matchSliceFn{Start: start, End: end, Step: step} -} - -func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { - // adjust indexes for negative values, reverse ordering - realStart, realEnd := f.Start, f.End - if realStart < 0 { - realStart = len(arr) + realStart - } - if realEnd < 0 { - realEnd = len(arr) + realEnd - } - if realEnd < realStart { - realEnd, realStart = realStart, realEnd // swap - } - // loop and gather - for idx := realStart; idx < realEnd; idx += f.Step { - f.next.call(arr[idx], ctx) - } - } -} - -// match anything -type matchAnyFn struct { - matchBase -} - -func newMatchAnyFn() *matchAnyFn { - return &matchAnyFn{} -} - -func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - for _, v := range tree.values { - f.next.call(v, ctx) - } - } -} - -// filter through union -type matchUnionFn struct { - Union []pathFn -} - -func (f *matchUnionFn) setNext(next pathFn) { - for _, fn := range f.Union { - fn.setNext(next) - } -} - -func (f *matchUnionFn) call(node interface{}, ctx *queryContext) { - for _, fn := range f.Union { - fn.call(node, ctx) - } -} - -// match every single last node in the tree -type matchRecursiveFn struct { - matchBase -} - -func newMatchRecursiveFn() *matchRecursiveFn { - return &matchRecursiveFn{} -} - -func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - var visit func(tree *TomlTree) - visit = func(tree *TomlTree) { - for _, v := range tree.values { - f.next.call(v, ctx) - switch node := v.(type) { - case *TomlTree: - visit(node) - case []*TomlTree: - for _, subtree := range node { - visit(subtree) - } - } - } - } - f.next.call(tree, ctx) - visit(tree) - } -} - -// match based on an externally provided functional filter -type matchFilterFn struct { - matchBase - Pos Position - Name string -} - -func newMatchFilterFn(name string, pos Position) *matchFilterFn { - return &matchFilterFn{Name: name, Pos: pos} -} - -func (f *matchFilterFn) call(node interface{}, ctx *queryContext) { - fn, ok := (*ctx.filters)[f.Name] - if !ok { - panic(fmt.Sprintf("%s: query context does not have filter '%s'", - f.Pos.String(), f.Name)) - } - switch castNode := tomlValueCheck(node, ctx).(type) { - case *TomlTree: - for _, v := range castNode.values { - if tv, ok := v.(*tomlValue); ok { - if fn(tv.value) { - f.next.call(v, ctx) - } - } else { - if fn(v) { - f.next.call(v, ctx) - } - } - } - case []interface{}: - for _, v := range castNode { - if fn(v) { - f.next.call(v, ctx) - } - } - } -} diff --git a/vendor/github.com/pelletier/go-toml/match_test.go b/vendor/github.com/pelletier/go-toml/match_test.go deleted file mode 100644 index b63654ad2f..0000000000 --- a/vendor/github.com/pelletier/go-toml/match_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package toml - -import ( - "fmt" - "testing" -) - -// dump path tree to a string -func pathString(root pathFn) string { - result := fmt.Sprintf("%T:", root) - switch fn := root.(type) { - case *terminatingFn: - result += "{}" - case *matchKeyFn: - result += fmt.Sprintf("{%s}", fn.Name) - result += pathString(fn.next) - case *matchIndexFn: - result += fmt.Sprintf("{%d}", fn.Idx) - result += pathString(fn.next) - case *matchSliceFn: - result += fmt.Sprintf("{%d:%d:%d}", - fn.Start, fn.End, fn.Step) - result += pathString(fn.next) - case *matchAnyFn: - result += "{}" - result += pathString(fn.next) - case *matchUnionFn: - result += "{[" - for _, v := range fn.Union { - result += pathString(v) + ", " - } - result += "]}" - case *matchRecursiveFn: - result += "{}" - result += pathString(fn.next) - case *matchFilterFn: - result += fmt.Sprintf("{%s}", fn.Name) - result += pathString(fn.next) - } - return result -} - -func assertPathMatch(t *testing.T, path, ref *Query) bool { - pathStr := pathString(path.root) - refStr := pathString(ref.root) - if pathStr != refStr { - t.Errorf("paths do not match") - t.Log("test:", pathStr) - t.Log("ref: ", refStr) - return false - } - return true -} - -func assertPath(t *testing.T, query string, ref *Query) { - path, _ := parseQuery(lexQuery(query)) - assertPathMatch(t, path, ref) -} - -func buildPath(parts ...pathFn) *Query { - query := newQuery() - for _, v := range parts { - query.appendPath(v) - } - return query -} - -func TestPathRoot(t *testing.T) { - assertPath(t, - "$", - buildPath( - // empty - )) -} - -func TestPathKey(t *testing.T) { - assertPath(t, - "$.foo", - buildPath( - newMatchKeyFn("foo"), - )) -} - -func TestPathBracketKey(t *testing.T) { - assertPath(t, - "$[foo]", - buildPath( - newMatchKeyFn("foo"), - )) -} - -func TestPathBracketStringKey(t *testing.T) { - assertPath(t, - "$['foo']", - buildPath( - newMatchKeyFn("foo"), - )) -} - -func TestPathIndex(t *testing.T) { - assertPath(t, - "$[123]", - buildPath( - newMatchIndexFn(123), - )) -} - -func TestPathSliceStart(t *testing.T) { - assertPath(t, - "$[123:]", - buildPath( - newMatchSliceFn(123, maxInt, 1), - )) -} - -func TestPathSliceStartEnd(t *testing.T) { - assertPath(t, - "$[123:456]", - buildPath( - newMatchSliceFn(123, 456, 1), - )) -} - -func TestPathSliceStartEndColon(t *testing.T) { - assertPath(t, - "$[123:456:]", - buildPath( - newMatchSliceFn(123, 456, 1), - )) -} - -func TestPathSliceStartStep(t *testing.T) { - assertPath(t, - "$[123::7]", - buildPath( - newMatchSliceFn(123, maxInt, 7), - )) -} - -func TestPathSliceEndStep(t *testing.T) { - assertPath(t, - "$[:456:7]", - buildPath( - newMatchSliceFn(0, 456, 7), - )) -} - -func TestPathSliceStep(t *testing.T) { - assertPath(t, - "$[::7]", - buildPath( - newMatchSliceFn(0, maxInt, 7), - )) -} - -func TestPathSliceAll(t *testing.T) { - assertPath(t, - "$[123:456:7]", - buildPath( - newMatchSliceFn(123, 456, 7), - )) -} - -func TestPathAny(t *testing.T) { - assertPath(t, - "$.*", - buildPath( - newMatchAnyFn(), - )) -} - -func TestPathUnion(t *testing.T) { - assertPath(t, - "$[foo, bar, baz]", - buildPath( - &matchUnionFn{[]pathFn{ - newMatchKeyFn("foo"), - newMatchKeyFn("bar"), - newMatchKeyFn("baz"), - }}, - )) -} - -func TestPathRecurse(t *testing.T) { - assertPath(t, - "$..*", - buildPath( - newMatchRecursiveFn(), - )) -} - -func TestPathFilterExpr(t *testing.T) { - assertPath(t, - "$[?('foo'),?(bar)]", - buildPath( - &matchUnionFn{[]pathFn{ - newMatchFilterFn("foo", Position{}), - newMatchFilterFn("bar", Position{}), - }}, - )) -} diff --git a/vendor/github.com/pelletier/go-toml/parser.go b/vendor/github.com/pelletier/go-toml/parser.go index 20e90a3ed7..d492a1e6ff 100644 --- a/vendor/github.com/pelletier/go-toml/parser.go +++ b/vendor/github.com/pelletier/go-toml/parser.go @@ -13,9 +13,9 @@ import ( ) type tomlParser struct { - flow chan token - tree *TomlTree - tokensBuffer []token + flowIdx int + flow []token + tree *Tree currentTable []string seenTableKeys []string } @@ -34,16 +34,10 @@ func (p *tomlParser) run() { } func (p *tomlParser) peek() *token { - if len(p.tokensBuffer) != 0 { - return &(p.tokensBuffer[0]) - } - - tok, ok := <-p.flow - if !ok { + if p.flowIdx >= len(p.flow) { return nil } - p.tokensBuffer = append(p.tokensBuffer, tok) - return &tok + return &p.flow[p.flowIdx] } func (p *tomlParser) assume(typ tokenType) { @@ -57,16 +51,12 @@ func (p *tomlParser) assume(typ tokenType) { } func (p *tomlParser) getToken() *token { - if len(p.tokensBuffer) != 0 { - tok := p.tokensBuffer[0] - p.tokensBuffer = p.tokensBuffer[1:] - return &tok - } - tok, ok := <-p.flow - if !ok { + tok := p.peek() + if tok == nil { return nil } - return &tok + p.flowIdx++ + return tok } func (p *tomlParser) parseStart() tomlParserStateFn { @@ -106,21 +96,21 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn { } p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries destTree := p.tree.GetPath(keys) - var array []*TomlTree + var array []*Tree if destTree == nil { - array = make([]*TomlTree, 0) - } else if target, ok := destTree.([]*TomlTree); ok && target != nil { - array = destTree.([]*TomlTree) + array = make([]*Tree, 0) + } else if target, ok := destTree.([]*Tree); ok && target != nil { + array = destTree.([]*Tree) } else { p.raiseError(key, "key %s is already assigned and not of type table array", key) } p.currentTable = keys // add a new tree to the end of the table array - newTree := newTomlTree() + newTree := newTree() newTree.position = startToken.Position array = append(array, newTree) - p.tree.SetPath(p.currentTable, array) + p.tree.SetPath(p.currentTable, "", false, array) // remove all keys that were children of this table array prefix := key.val + "." @@ -183,11 +173,11 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { } // find the table to assign, looking out for arrays of tables - var targetNode *TomlTree + var targetNode *Tree switch node := p.tree.GetPath(tableKey).(type) { - case []*TomlTree: + case []*Tree: targetNode = node[len(node)-1] - case *TomlTree: + case *Tree: targetNode = node default: p.raiseError(key, "Unknown table type for path: %s", @@ -212,10 +202,10 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { var toInsert interface{} switch value.(type) { - case *TomlTree, []*TomlTree: + case *Tree, []*Tree: toInsert = value default: - toInsert = &tomlValue{value, key.Position} + toInsert = &tomlValue{value: value, position: key.Position} } targetNode.values[keyVal] = toInsert return p.parseStart @@ -289,8 +279,8 @@ func tokenIsComma(t *token) bool { return t != nil && t.typ == tokenComma } -func (p *tomlParser) parseInlineTable() *TomlTree { - tree := newTomlTree() +func (p *tomlParser) parseInlineTable() *Tree { + tree := newTree() var previous *token Loop: for { @@ -309,7 +299,7 @@ Loop: key := p.getToken() p.assume(tokenEqual) value := p.parseRvalue() - tree.Set(key.val, value) + tree.Set(key.val, "", false, value) case tokenComma: if previous == nil { p.raiseError(follow, "inline table cannot start with a comma") @@ -360,27 +350,27 @@ func (p *tomlParser) parseArray() interface{} { p.getToken() } } - // An array of TomlTrees is actually an array of inline + // An array of Trees is actually an array of inline // tables, which is a shorthand for a table array. If the - // array was not converted from []interface{} to []*TomlTree, + // array was not converted from []interface{} to []*Tree, // the two notations would not be equivalent. - if arrayType == reflect.TypeOf(newTomlTree()) { - tomlArray := make([]*TomlTree, len(array)) + if arrayType == reflect.TypeOf(newTree()) { + tomlArray := make([]*Tree, len(array)) for i, v := range array { - tomlArray[i] = v.(*TomlTree) + tomlArray[i] = v.(*Tree) } return tomlArray } return array } -func parseToml(flow chan token) *TomlTree { - result := newTomlTree() +func parseToml(flow []token) *Tree { + result := newTree() result.position = Position{1, 1} parser := &tomlParser{ + flowIdx: 0, flow: flow, tree: result, - tokensBuffer: make([]token, 0), currentTable: make([]string, 0), seenTableKeys: make([]string, 0), } diff --git a/vendor/github.com/pelletier/go-toml/parser_test.go b/vendor/github.com/pelletier/go-toml/parser_test.go index 58aae203a6..bc7903c8d4 100644 --- a/vendor/github.com/pelletier/go-toml/parser_test.go +++ b/vendor/github.com/pelletier/go-toml/parser_test.go @@ -9,7 +9,7 @@ import ( "github.com/davecgh/go-spew/spew" ) -func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref map[string]interface{}) { +func assertSubTree(t *testing.T, path []string, tree *Tree, err error, ref map[string]interface{}) { if err != nil { t.Error("Non-nil error:", err.Error()) return @@ -20,12 +20,12 @@ func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref m // NOTE: directly access key instead of resolve by path // NOTE: see TestSpecialKV switch node := tree.GetPath([]string{k}).(type) { - case []*TomlTree: + case []*Tree: t.Log("\tcomparing key", nextPath, "by array iteration") for idx, item := range node { assertSubTree(t, nextPath, item, err, v.([]map[string]interface{})[idx]) } - case *TomlTree: + case *Tree: t.Log("\tcomparing key", nextPath, "by subtree assestion") assertSubTree(t, nextPath, node, err, v.(map[string]interface{})) default: @@ -37,16 +37,16 @@ func assertSubTree(t *testing.T, path []string, tree *TomlTree, err error, ref m } } -func assertTree(t *testing.T, tree *TomlTree, err error, ref map[string]interface{}) { +func assertTree(t *testing.T, tree *Tree, err error, ref map[string]interface{}) { t.Log("Asserting tree:\n", spew.Sdump(tree)) assertSubTree(t, []string{}, tree, err, ref) t.Log("Finished tree assertion.") } func TestCreateSubTree(t *testing.T) { - tree := newTomlTree() + tree := newTree() tree.createSubTree([]string{"a", "b", "c"}, Position{}) - tree.Set("a.b.c", 42) + tree.Set("a.b.c", "", false, 42) if tree.Get("a.b.c") != 42 { t.Fail() } @@ -652,7 +652,7 @@ func TestTomlValueStringRepresentation(t *testing.T) { "[\"gamma\",\"delta\"]"}, {nil, ""}, } { - result, err := tomlValueStringRepresentation(item.Value) + result, err := tomlValueStringRepresentation(item.Value, "", false) if err != nil { t.Errorf("Test %d - unexpected error: %s", idx, err) } diff --git a/vendor/github.com/pelletier/go-toml/query.go b/vendor/github.com/pelletier/go-toml/query.go deleted file mode 100644 index 307a1ecabc..0000000000 --- a/vendor/github.com/pelletier/go-toml/query.go +++ /dev/null @@ -1,153 +0,0 @@ -package toml - -import ( - "time" -) - -// NodeFilterFn represents a user-defined filter function, for use with -// Query.SetFilter(). -// -// The return value of the function must indicate if 'node' is to be included -// at this stage of the TOML path. Returning true will include the node, and -// returning false will exclude it. -// -// NOTE: Care should be taken to write script callbacks such that they are safe -// to use from multiple goroutines. -type NodeFilterFn func(node interface{}) bool - -// QueryResult is the result of Executing a Query. -type QueryResult struct { - items []interface{} - positions []Position -} - -// appends a value/position pair to the result set. -func (r *QueryResult) appendResult(node interface{}, pos Position) { - r.items = append(r.items, node) - r.positions = append(r.positions, pos) -} - -// Values is a set of values within a QueryResult. The order of values is not -// guaranteed to be in document order, and may be different each time a query is -// executed. -func (r QueryResult) Values() []interface{} { - values := make([]interface{}, len(r.items)) - for i, v := range r.items { - o, ok := v.(*tomlValue) - if ok { - values[i] = o.value - } else { - values[i] = v - } - } - return values -} - -// Positions is a set of positions for values within a QueryResult. Each index -// in Positions() corresponds to the entry in Value() of the same index. -func (r QueryResult) Positions() []Position { - return r.positions -} - -// runtime context for executing query paths -type queryContext struct { - result *QueryResult - filters *map[string]NodeFilterFn - lastPosition Position -} - -// generic path functor interface -type pathFn interface { - setNext(next pathFn) - call(node interface{}, ctx *queryContext) -} - -// A Query is the representation of a compiled TOML path. A Query is safe -// for concurrent use by multiple goroutines. -type Query struct { - root pathFn - tail pathFn - filters *map[string]NodeFilterFn -} - -func newQuery() *Query { - return &Query{ - root: nil, - tail: nil, - filters: &defaultFilterFunctions, - } -} - -func (q *Query) appendPath(next pathFn) { - if q.root == nil { - q.root = next - } else { - q.tail.setNext(next) - } - q.tail = next - next.setNext(newTerminatingFn()) // init the next functor -} - -// CompileQuery compiles a TOML path expression. The returned Query can be used -// to match elements within a TomlTree and its descendants. -func CompileQuery(path string) (*Query, error) { - return parseQuery(lexQuery(path)) -} - -// Execute executes a query against a TomlTree, and returns the result of the query. -func (q *Query) Execute(tree *TomlTree) *QueryResult { - result := &QueryResult{ - items: []interface{}{}, - positions: []Position{}, - } - if q.root == nil { - result.appendResult(tree, tree.GetPosition("")) - } else { - ctx := &queryContext{ - result: result, - filters: q.filters, - } - q.root.call(tree, ctx) - } - return result -} - -// SetFilter sets a user-defined filter function. These may be used inside -// "?(..)" query expressions to filter TOML document elements within a query. -func (q *Query) SetFilter(name string, fn NodeFilterFn) { - if q.filters == &defaultFilterFunctions { - // clone the static table - q.filters = &map[string]NodeFilterFn{} - for k, v := range defaultFilterFunctions { - (*q.filters)[k] = v - } - } - (*q.filters)[name] = fn -} - -var defaultFilterFunctions = map[string]NodeFilterFn{ - "tree": func(node interface{}) bool { - _, ok := node.(*TomlTree) - return ok - }, - "int": func(node interface{}) bool { - _, ok := node.(int64) - return ok - }, - "float": func(node interface{}) bool { - _, ok := node.(float64) - return ok - }, - "string": func(node interface{}) bool { - _, ok := node.(string) - return ok - }, - "time": func(node interface{}) bool { - _, ok := node.(time.Time) - return ok - }, - "bool": func(node interface{}) bool { - _, ok := node.(bool) - return ok - }, -} diff --git a/vendor/github.com/pelletier/go-toml/query_test.go b/vendor/github.com/pelletier/go-toml/query_test.go deleted file mode 100644 index 0d9f3832b9..0000000000 --- a/vendor/github.com/pelletier/go-toml/query_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package toml - -import ( - "testing" -) - -func assertArrayContainsInAnyOrder(t *testing.T, array []interface{}, objects ...interface{}) { - if len(array) != len(objects) { - t.Fatalf("array contains %d objects but %d are expected", len(array), len(objects)) - } - - for _, o := range objects { - found := false - for _, a := range array { - if a == o { - found = true - break - } - } - if !found { - t.Fatal(o, "not found in array", array) - } - } -} - -func TestQueryExample(t *testing.T) { - config, _ := Load(` - [[book]] - title = "The Stand" - author = "Stephen King" - [[book]] - title = "For Whom the Bell Tolls" - author = "Ernest Hemmingway" - [[book]] - title = "Neuromancer" - author = "William Gibson" - `) - - authors, _ := config.Query("$.book.author") - names := authors.Values() - if len(names) != 3 { - t.Fatalf("query should return 3 names but returned %d", len(names)) - } - assertArrayContainsInAnyOrder(t, names, "Stephen King", "Ernest Hemmingway", "William Gibson") -} - -func TestQueryReadmeExample(t *testing.T) { - config, _ := Load(` -[postgres] -user = "pelletier" -password = "mypassword" -`) - results, _ := config.Query("$..[user,password]") - values := results.Values() - if len(values) != 2 { - t.Fatalf("query should return 2 values but returned %d", len(values)) - } - assertArrayContainsInAnyOrder(t, values, "pelletier", "mypassword") -} - -func TestQueryPathNotPresent(t *testing.T) { - config, _ := Load(`a = "hello"`) - results, err := config.Query("$.foo.bar") - if err != nil { - t.Fatalf("err should be nil. got %s instead", err) - } - if len(results.items) != 0 { - t.Fatalf("no items should be matched. %d matched instead", len(results.items)) - } -} diff --git a/vendor/github.com/pelletier/go-toml/querylexer.go b/vendor/github.com/pelletier/go-toml/querylexer.go deleted file mode 100644 index 960681d0ff..0000000000 --- a/vendor/github.com/pelletier/go-toml/querylexer.go +++ /dev/null @@ -1,356 +0,0 @@ -// TOML JSONPath lexer. -// -// Written using the principles developed by Rob Pike in -// http://www.youtube.com/watch?v=HxaD_trXwRE - -package toml - -import ( - "fmt" - "strconv" - "strings" - "unicode/utf8" -) - -// Lexer state function -type queryLexStateFn func() queryLexStateFn - -// Lexer definition -type queryLexer struct { - input string - start int - pos int - width int - tokens chan token - depth int - line int - col int - stringTerm string -} - -func (l *queryLexer) run() { - for state := l.lexVoid; state != nil; { - state = state() - } - close(l.tokens) -} - -func (l *queryLexer) nextStart() { - // iterate by runes (utf8 characters) - // search for newlines and advance line/col counts - for i := l.start; i < l.pos; { - r, width := utf8.DecodeRuneInString(l.input[i:]) - if r == '\n' { - l.line++ - l.col = 1 - } else { - l.col++ - } - i += width - } - // advance start position to next token - l.start = l.pos -} - -func (l *queryLexer) emit(t tokenType) { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: t, - val: l.input[l.start:l.pos], - } - l.nextStart() -} - -func (l *queryLexer) emitWithValue(t tokenType, value string) { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: t, - val: value, - } - l.nextStart() -} - -func (l *queryLexer) next() rune { - if l.pos >= len(l.input) { - l.width = 0 - return eof - } - var r rune - r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) - l.pos += l.width - return r -} - -func (l *queryLexer) ignore() { - l.nextStart() -} - -func (l *queryLexer) backup() { - l.pos -= l.width -} - -func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: tokenError, - val: fmt.Sprintf(format, args...), - } - return nil -} - -func (l *queryLexer) peek() rune { - r := l.next() - l.backup() - return r -} - -func (l *queryLexer) accept(valid string) bool { - if strings.ContainsRune(valid, l.next()) { - return true - } - l.backup() - return false -} - -func (l *queryLexer) follow(next string) bool { - return strings.HasPrefix(l.input[l.pos:], next) -} - -func (l *queryLexer) lexVoid() queryLexStateFn { - for { - next := l.peek() - switch next { - case '$': - l.pos++ - l.emit(tokenDollar) - continue - case '.': - if l.follow("..") { - l.pos += 2 - l.emit(tokenDotDot) - } else { - l.pos++ - l.emit(tokenDot) - } - continue - case '[': - l.pos++ - l.emit(tokenLeftBracket) - continue - case ']': - l.pos++ - l.emit(tokenRightBracket) - continue - case ',': - l.pos++ - l.emit(tokenComma) - continue - case '*': - l.pos++ - l.emit(tokenStar) - continue - case '(': - l.pos++ - l.emit(tokenLeftParen) - continue - case ')': - l.pos++ - l.emit(tokenRightParen) - continue - case '?': - l.pos++ - l.emit(tokenQuestion) - continue - case ':': - l.pos++ - l.emit(tokenColon) - continue - case '\'': - l.ignore() - l.stringTerm = string(next) - return l.lexString - case '"': - l.ignore() - l.stringTerm = string(next) - return l.lexString - } - - if isSpace(next) { - l.next() - l.ignore() - continue - } - - if isAlphanumeric(next) { - return l.lexKey - } - - if next == '+' || next == '-' || isDigit(next) { - return l.lexNumber - } - - if l.next() == eof { - break - } - - return l.errorf("unexpected char: '%v'", next) - } - l.emit(tokenEOF) - return nil -} - -func (l *queryLexer) lexKey() queryLexStateFn { - for { - next := l.peek() - if !isAlphanumeric(next) { - l.emit(tokenKey) - return l.lexVoid - } - - if l.next() == eof { - break - } - } - l.emit(tokenEOF) - return nil -} - -func (l *queryLexer) lexString() queryLexStateFn { - l.pos++ - l.ignore() - growingString := "" - - for { - if l.follow(l.stringTerm) { - l.emitWithValue(tokenString, growingString) - l.pos++ - l.ignore() - return l.lexVoid - } - - if l.follow("\\\"") { - l.pos++ - growingString += "\"" - } else if l.follow("\\'") { - l.pos++ - growingString += "'" - } else if l.follow("\\n") { - l.pos++ - growingString += "\n" - } else if l.follow("\\b") { - l.pos++ - growingString += "\b" - } else if l.follow("\\f") { - l.pos++ - growingString += "\f" - } else if l.follow("\\/") { - l.pos++ - growingString += "/" - } else if l.follow("\\t") { - l.pos++ - growingString += "\t" - } else if l.follow("\\r") { - l.pos++ - growingString += "\r" - } else if l.follow("\\\\") { - l.pos++ - growingString += "\\" - } else if l.follow("\\u") { - l.pos += 2 - code := "" - for i := 0; i < 4; i++ { - c := l.peek() - l.pos++ - if !isHexDigit(c) { - return l.errorf("unfinished unicode escape") - } - code = code + string(c) - } - l.pos-- - intcode, err := strconv.ParseInt(code, 16, 32) - if err != nil { - return l.errorf("invalid unicode escape: \\u" + code) - } - growingString += string(rune(intcode)) - } else if l.follow("\\U") { - l.pos += 2 - code := "" - for i := 0; i < 8; i++ { - c := l.peek() - l.pos++ - if !isHexDigit(c) { - return l.errorf("unfinished unicode escape") - } - code = code + string(c) - } - l.pos-- - intcode, err := strconv.ParseInt(code, 16, 32) - if err != nil { - return l.errorf("invalid unicode escape: \\u" + code) - } - growingString += string(rune(intcode)) - } else if l.follow("\\") { - l.pos++ - return l.errorf("invalid escape sequence: \\" + string(l.peek())) - } else { - growingString += string(l.peek()) - } - - if l.next() == eof { - break - } - } - - return l.errorf("unclosed string") -} - -func (l *queryLexer) lexNumber() queryLexStateFn { - l.ignore() - if !l.accept("+") { - l.accept("-") - } - pointSeen := false - digitSeen := false - for { - next := l.next() - if next == '.' { - if pointSeen { - return l.errorf("cannot have two dots in one float") - } - if !isDigit(l.peek()) { - return l.errorf("float cannot end with a dot") - } - pointSeen = true - } else if isDigit(next) { - digitSeen = true - } else { - l.backup() - break - } - if pointSeen && !digitSeen { - return l.errorf("cannot start float with a dot") - } - } - - if !digitSeen { - return l.errorf("no digit in that number") - } - if pointSeen { - l.emit(tokenFloat) - } else { - l.emit(tokenInteger) - } - return l.lexVoid -} - -// Entry point -func lexQuery(input string) chan token { - l := &queryLexer{ - input: input, - tokens: make(chan token), - line: 1, - col: 1, - } - go l.run() - return l.tokens -} diff --git a/vendor/github.com/pelletier/go-toml/querylexer_test.go b/vendor/github.com/pelletier/go-toml/querylexer_test.go deleted file mode 100644 index 2d0803ff70..0000000000 --- a/vendor/github.com/pelletier/go-toml/querylexer_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package toml - -import ( - "testing" -) - -func testQLFlow(t *testing.T, input string, expectedFlow []token) { - ch := lexQuery(input) - for idx, expected := range expectedFlow { - token := <-ch - if token != expected { - t.Log("While testing #", idx, ":", input) - t.Log("compared (got)", token, "to (expected)", expected) - t.Log("\tvalue:", token.val, "<->", expected.val) - t.Log("\tvalue as bytes:", []byte(token.val), "<->", []byte(expected.val)) - t.Log("\ttype:", token.typ.String(), "<->", expected.typ.String()) - t.Log("\tline:", token.Line, "<->", expected.Line) - t.Log("\tcolumn:", token.Col, "<->", expected.Col) - t.Log("compared", token, "to", expected) - t.FailNow() - } - } - - tok, ok := <-ch - if ok { - t.Log("channel is not closed!") - t.Log(len(ch)+1, "tokens remaining:") - - t.Log("token ->", tok) - for token := range ch { - t.Log("token ->", token) - } - t.FailNow() - } -} - -func TestLexSpecialChars(t *testing.T) { - testQLFlow(t, " .$[]..()?*", []token{ - {Position{1, 2}, tokenDot, "."}, - {Position{1, 3}, tokenDollar, "$"}, - {Position{1, 4}, tokenLeftBracket, "["}, - {Position{1, 5}, tokenRightBracket, "]"}, - {Position{1, 6}, tokenDotDot, ".."}, - {Position{1, 8}, tokenLeftParen, "("}, - {Position{1, 9}, tokenRightParen, ")"}, - {Position{1, 10}, tokenQuestion, "?"}, - {Position{1, 11}, tokenStar, "*"}, - {Position{1, 12}, tokenEOF, ""}, - }) -} - -func TestLexString(t *testing.T) { - testQLFlow(t, "'foo\n'", []token{ - {Position{1, 2}, tokenString, "foo\n"}, - {Position{2, 2}, tokenEOF, ""}, - }) -} - -func TestLexDoubleString(t *testing.T) { - testQLFlow(t, `"bar"`, []token{ - {Position{1, 2}, tokenString, "bar"}, - {Position{1, 6}, tokenEOF, ""}, - }) -} - -func TestLexStringEscapes(t *testing.T) { - testQLFlow(t, `"foo \" \' \b \f \/ \t \r \\ \u03A9 \U00012345 \n bar"`, []token{ - {Position{1, 2}, tokenString, "foo \" ' \b \f / \t \r \\ \u03A9 \U00012345 \n bar"}, - {Position{1, 55}, tokenEOF, ""}, - }) -} - -func TestLexStringUnfinishedUnicode4(t *testing.T) { - testQLFlow(t, `"\u000"`, []token{ - {Position{1, 2}, tokenError, "unfinished unicode escape"}, - }) -} - -func TestLexStringUnfinishedUnicode8(t *testing.T) { - testQLFlow(t, `"\U0000"`, []token{ - {Position{1, 2}, tokenError, "unfinished unicode escape"}, - }) -} - -func TestLexStringInvalidEscape(t *testing.T) { - testQLFlow(t, `"\x"`, []token{ - {Position{1, 2}, tokenError, "invalid escape sequence: \\x"}, - }) -} - -func TestLexStringUnfinished(t *testing.T) { - testQLFlow(t, `"bar`, []token{ - {Position{1, 2}, tokenError, "unclosed string"}, - }) -} - -func TestLexKey(t *testing.T) { - testQLFlow(t, "foo", []token{ - {Position{1, 1}, tokenKey, "foo"}, - {Position{1, 4}, tokenEOF, ""}, - }) -} - -func TestLexRecurse(t *testing.T) { - testQLFlow(t, "$..*", []token{ - {Position{1, 1}, tokenDollar, "$"}, - {Position{1, 2}, tokenDotDot, ".."}, - {Position{1, 4}, tokenStar, "*"}, - {Position{1, 5}, tokenEOF, ""}, - }) -} - -func TestLexBracketKey(t *testing.T) { - testQLFlow(t, "$[foo]", []token{ - {Position{1, 1}, tokenDollar, "$"}, - {Position{1, 2}, tokenLeftBracket, "["}, - {Position{1, 3}, tokenKey, "foo"}, - {Position{1, 6}, tokenRightBracket, "]"}, - {Position{1, 7}, tokenEOF, ""}, - }) -} - -func TestLexSpace(t *testing.T) { - testQLFlow(t, "foo bar baz", []token{ - {Position{1, 1}, tokenKey, "foo"}, - {Position{1, 5}, tokenKey, "bar"}, - {Position{1, 9}, tokenKey, "baz"}, - {Position{1, 12}, tokenEOF, ""}, - }) -} - -func TestLexInteger(t *testing.T) { - testQLFlow(t, "100 +200 -300", []token{ - {Position{1, 1}, tokenInteger, "100"}, - {Position{1, 5}, tokenInteger, "+200"}, - {Position{1, 10}, tokenInteger, "-300"}, - {Position{1, 14}, tokenEOF, ""}, - }) -} - -func TestLexFloat(t *testing.T) { - testQLFlow(t, "100.0 +200.0 -300.0", []token{ - {Position{1, 1}, tokenFloat, "100.0"}, - {Position{1, 7}, tokenFloat, "+200.0"}, - {Position{1, 14}, tokenFloat, "-300.0"}, - {Position{1, 20}, tokenEOF, ""}, - }) -} - -func TestLexFloatWithMultipleDots(t *testing.T) { - testQLFlow(t, "4.2.", []token{ - {Position{1, 1}, tokenError, "cannot have two dots in one float"}, - }) -} - -func TestLexFloatLeadingDot(t *testing.T) { - testQLFlow(t, "+.1", []token{ - {Position{1, 1}, tokenError, "cannot start float with a dot"}, - }) -} - -func TestLexFloatWithTrailingDot(t *testing.T) { - testQLFlow(t, "42.", []token{ - {Position{1, 1}, tokenError, "float cannot end with a dot"}, - }) -} - -func TestLexNumberWithoutDigit(t *testing.T) { - testQLFlow(t, "+", []token{ - {Position{1, 1}, tokenError, "no digit in that number"}, - }) -} - -func TestLexUnknown(t *testing.T) { - testQLFlow(t, "^", []token{ - {Position{1, 1}, tokenError, "unexpected char: '94'"}, - }) -} diff --git a/vendor/github.com/pelletier/go-toml/queryparser.go b/vendor/github.com/pelletier/go-toml/queryparser.go deleted file mode 100644 index 1cbfc83b26..0000000000 --- a/vendor/github.com/pelletier/go-toml/queryparser.go +++ /dev/null @@ -1,275 +0,0 @@ -/* - Based on the "jsonpath" spec/concept. - - http://goessner.net/articles/JsonPath/ - https://code.google.com/p/json-path/ -*/ - -package toml - -import ( - "fmt" -) - -const maxInt = int(^uint(0) >> 1) - -type queryParser struct { - flow chan token - tokensBuffer []token - query *Query - union []pathFn - err error -} - -type queryParserStateFn func() queryParserStateFn - -// Formats and panics an error message based on a token -func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn { - p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...) - return nil // trigger parse to end -} - -func (p *queryParser) run() { - for state := p.parseStart; state != nil; { - state = state() - } -} - -func (p *queryParser) backup(tok *token) { - p.tokensBuffer = append(p.tokensBuffer, *tok) -} - -func (p *queryParser) peek() *token { - if len(p.tokensBuffer) != 0 { - return &(p.tokensBuffer[0]) - } - - tok, ok := <-p.flow - if !ok { - return nil - } - p.backup(&tok) - return &tok -} - -func (p *queryParser) lookahead(types ...tokenType) bool { - result := true - buffer := []token{} - - for _, typ := range types { - tok := p.getToken() - if tok == nil { - result = false - break - } - buffer = append(buffer, *tok) - if tok.typ != typ { - result = false - break - } - } - // add the tokens back to the buffer, and return - p.tokensBuffer = append(p.tokensBuffer, buffer...) - return result -} - -func (p *queryParser) getToken() *token { - if len(p.tokensBuffer) != 0 { - tok := p.tokensBuffer[0] - p.tokensBuffer = p.tokensBuffer[1:] - return &tok - } - tok, ok := <-p.flow - if !ok { - return nil - } - return &tok -} - -func (p *queryParser) parseStart() queryParserStateFn { - tok := p.getToken() - - if tok == nil || tok.typ == tokenEOF { - return nil - } - - if tok.typ != tokenDollar { - return p.parseError(tok, "Expected '$' at start of expression") - } - - return p.parseMatchExpr -} - -// handle '.' prefix, '[]', and '..' -func (p *queryParser) parseMatchExpr() queryParserStateFn { - tok := p.getToken() - switch tok.typ { - case tokenDotDot: - p.query.appendPath(&matchRecursiveFn{}) - // nested parse for '..' - tok := p.getToken() - switch tok.typ { - case tokenKey: - p.query.appendPath(newMatchKeyFn(tok.val)) - return p.parseMatchExpr - case tokenLeftBracket: - return p.parseBracketExpr - case tokenStar: - // do nothing - the recursive predicate is enough - return p.parseMatchExpr - } - - case tokenDot: - // nested parse for '.' - tok := p.getToken() - switch tok.typ { - case tokenKey: - p.query.appendPath(newMatchKeyFn(tok.val)) - return p.parseMatchExpr - case tokenStar: - p.query.appendPath(&matchAnyFn{}) - return p.parseMatchExpr - } - - case tokenLeftBracket: - return p.parseBracketExpr - - case tokenEOF: - return nil // allow EOF at this stage - } - return p.parseError(tok, "expected match expression") -} - -func (p *queryParser) parseBracketExpr() queryParserStateFn { - if p.lookahead(tokenInteger, tokenColon) { - return p.parseSliceExpr - } - if p.peek().typ == tokenColon { - return p.parseSliceExpr - } - return p.parseUnionExpr -} - -func (p *queryParser) parseUnionExpr() queryParserStateFn { - var tok *token - - // this state can be traversed after some sub-expressions - // so be careful when setting up state in the parser - if p.union == nil { - p.union = []pathFn{} - } - -loop: // labeled loop for easy breaking - for { - if len(p.union) > 0 { - // parse delimiter or terminator - tok = p.getToken() - switch tok.typ { - case tokenComma: - // do nothing - case tokenRightBracket: - break loop - default: - return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val) - } - } - - // parse sub expression - tok = p.getToken() - switch tok.typ { - case tokenInteger: - p.union = append(p.union, newMatchIndexFn(tok.Int())) - case tokenKey: - p.union = append(p.union, newMatchKeyFn(tok.val)) - case tokenString: - p.union = append(p.union, newMatchKeyFn(tok.val)) - case tokenQuestion: - return p.parseFilterExpr - default: - return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union)) - } - } - - // if there is only one sub-expression, use that instead - if len(p.union) == 1 { - p.query.appendPath(p.union[0]) - } else { - p.query.appendPath(&matchUnionFn{p.union}) - } - - p.union = nil // clear out state - return p.parseMatchExpr -} - -func (p *queryParser) parseSliceExpr() queryParserStateFn { - // init slice to grab all elements - start, end, step := 0, maxInt, 1 - - // parse optional start - tok := p.getToken() - if tok.typ == tokenInteger { - start = tok.Int() - tok = p.getToken() - } - if tok.typ != tokenColon { - return p.parseError(tok, "expected ':'") - } - - // parse optional end - tok = p.getToken() - if tok.typ == tokenInteger { - end = tok.Int() - tok = p.getToken() - } - if tok.typ == tokenRightBracket { - p.query.appendPath(newMatchSliceFn(start, end, step)) - return p.parseMatchExpr - } - if tok.typ != tokenColon { - return p.parseError(tok, "expected ']' or ':'") - } - - // parse optional step - tok = p.getToken() - if tok.typ == tokenInteger { - step = tok.Int() - if step < 0 { - return p.parseError(tok, "step must be a positive value") - } - tok = p.getToken() - } - if tok.typ != tokenRightBracket { - return p.parseError(tok, "expected ']'") - } - - p.query.appendPath(newMatchSliceFn(start, end, step)) - return p.parseMatchExpr -} - -func (p *queryParser) parseFilterExpr() queryParserStateFn { - tok := p.getToken() - if tok.typ != tokenLeftParen { - return p.parseError(tok, "expected left-parenthesis for filter expression") - } - tok = p.getToken() - if tok.typ != tokenKey && tok.typ != tokenString { - return p.parseError(tok, "expected key or string for filter funciton name") - } - name := tok.val - tok = p.getToken() - if tok.typ != tokenRightParen { - return p.parseError(tok, "expected right-parenthesis for filter expression") - } - p.union = append(p.union, newMatchFilterFn(name, tok.Position)) - return p.parseUnionExpr -} - -func parseQuery(flow chan token) (*Query, error) { - parser := &queryParser{ - flow: flow, - tokensBuffer: []token{}, - query: newQuery(), - } - parser.run() - return parser.query, parser.err -} diff --git a/vendor/github.com/pelletier/go-toml/queryparser_test.go b/vendor/github.com/pelletier/go-toml/queryparser_test.go deleted file mode 100644 index b2b85cefdc..0000000000 --- a/vendor/github.com/pelletier/go-toml/queryparser_test.go +++ /dev/null @@ -1,483 +0,0 @@ -package toml - -import ( - "fmt" - "io/ioutil" - "sort" - "strings" - "testing" - "time" -) - -type queryTestNode struct { - value interface{} - position Position -} - -func valueString(root interface{}) string { - result := "" //fmt.Sprintf("%T:", root) - switch node := root.(type) { - case *tomlValue: - return valueString(node.value) - case *QueryResult: - items := []string{} - for i, v := range node.Values() { - items = append(items, fmt.Sprintf("%s:%s", - node.Positions()[i].String(), valueString(v))) - } - sort.Strings(items) - result = "[" + strings.Join(items, ", ") + "]" - case queryTestNode: - result = fmt.Sprintf("%s:%s", - node.position.String(), valueString(node.value)) - case []interface{}: - items := []string{} - for _, v := range node { - items = append(items, valueString(v)) - } - sort.Strings(items) - result = "[" + strings.Join(items, ", ") + "]" - case *TomlTree: - // workaround for unreliable map key ordering - items := []string{} - for _, k := range node.Keys() { - v := node.GetPath([]string{k}) - items = append(items, k+":"+valueString(v)) - } - sort.Strings(items) - result = "{" + strings.Join(items, ", ") + "}" - case map[string]interface{}: - // workaround for unreliable map key ordering - items := []string{} - for k, v := range node { - items = append(items, k+":"+valueString(v)) - } - sort.Strings(items) - result = "{" + strings.Join(items, ", ") + "}" - case int64: - result += fmt.Sprintf("%d", node) - case string: - result += "'" + node + "'" - case float64: - result += fmt.Sprintf("%f", node) - case bool: - result += fmt.Sprintf("%t", node) - case time.Time: - result += fmt.Sprintf("'%v'", node) - } - return result -} - -func assertValue(t *testing.T, result, ref interface{}) { - pathStr := valueString(result) - refStr := valueString(ref) - if pathStr != refStr { - t.Errorf("values do not match") - t.Log("test:", pathStr) - t.Log("ref: ", refStr) - } -} - -func assertQueryPositions(t *testing.T, toml, query string, ref []interface{}) { - tree, err := Load(toml) - if err != nil { - t.Errorf("Non-nil toml parse error: %v", err) - return - } - q, err := CompileQuery(query) - if err != nil { - t.Error(err) - return - } - results := q.Execute(tree) - assertValue(t, results, ref) -} - -func TestQueryRoot(t *testing.T) { - assertQueryPositions(t, - "a = 42", - "$", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "a": int64(42), - }, Position{1, 1}, - }, - }) -} - -func TestQueryKey(t *testing.T) { - assertQueryPositions(t, - "[foo]\na = 42", - "$.foo.a", - []interface{}{ - queryTestNode{ - int64(42), Position{2, 1}, - }, - }) -} - -func TestQueryKeyString(t *testing.T) { - assertQueryPositions(t, - "[foo]\na = 42", - "$.foo['a']", - []interface{}{ - queryTestNode{ - int64(42), Position{2, 1}, - }, - }) -} - -func TestQueryIndex(t *testing.T) { - assertQueryPositions(t, - "[foo]\na = [1,2,3,4,5,6,7,8,9,0]", - "$.foo.a[5]", - []interface{}{ - queryTestNode{ - int64(6), Position{2, 1}, - }, - }) -} - -func TestQuerySliceRange(t *testing.T) { - assertQueryPositions(t, - "[foo]\na = [1,2,3,4,5,6,7,8,9,0]", - "$.foo.a[0:5]", - []interface{}{ - queryTestNode{ - int64(1), Position{2, 1}, - }, - queryTestNode{ - int64(2), Position{2, 1}, - }, - queryTestNode{ - int64(3), Position{2, 1}, - }, - queryTestNode{ - int64(4), Position{2, 1}, - }, - queryTestNode{ - int64(5), Position{2, 1}, - }, - }) -} - -func TestQuerySliceStep(t *testing.T) { - assertQueryPositions(t, - "[foo]\na = [1,2,3,4,5,6,7,8,9,0]", - "$.foo.a[0:5:2]", - []interface{}{ - queryTestNode{ - int64(1), Position{2, 1}, - }, - queryTestNode{ - int64(3), Position{2, 1}, - }, - queryTestNode{ - int64(5), Position{2, 1}, - }, - }) -} - -func TestQueryAny(t *testing.T) { - assertQueryPositions(t, - "[foo.bar]\na=1\nb=2\n[foo.baz]\na=3\nb=4", - "$.foo.*", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, Position{4, 1}, - }, - }) -} -func TestQueryUnionSimple(t *testing.T) { - assertQueryPositions(t, - "[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6", - "$.*[bar,foo]", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, Position{4, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(5), - "b": int64(6), - }, Position{7, 1}, - }, - }) -} - -func TestQueryRecursionAll(t *testing.T) { - assertQueryPositions(t, - "[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6", - "$..*", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, - }, - "baz": map[string]interface{}{ - "foo": map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, - }, - "gorf": map[string]interface{}{ - "foo": map[string]interface{}{ - "a": int64(5), - "b": int64(6), - }, - }, - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "bar": map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, Position{1, 1}, - }, - queryTestNode{ - int64(1), Position{2, 1}, - }, - queryTestNode{ - int64(2), Position{3, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "foo": map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, - }, Position{4, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, Position{4, 1}, - }, - queryTestNode{ - int64(3), Position{5, 1}, - }, - queryTestNode{ - int64(4), Position{6, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "foo": map[string]interface{}{ - "a": int64(5), - "b": int64(6), - }, - }, Position{7, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(5), - "b": int64(6), - }, Position{7, 1}, - }, - queryTestNode{ - int64(5), Position{8, 1}, - }, - queryTestNode{ - int64(6), Position{9, 1}, - }, - }) -} - -func TestQueryRecursionUnionSimple(t *testing.T) { - assertQueryPositions(t, - "[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6", - "$..['foo','bar']", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "bar": map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(3), - "b": int64(4), - }, Position{4, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(1), - "b": int64(2), - }, Position{1, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "a": int64(5), - "b": int64(6), - }, Position{7, 1}, - }, - }) -} - -func TestQueryFilterFn(t *testing.T) { - buff, err := ioutil.ReadFile("example.toml") - if err != nil { - t.Error(err) - return - } - - assertQueryPositions(t, string(buff), - "$..[?(int)]", - []interface{}{ - queryTestNode{ - int64(8001), Position{13, 1}, - }, - queryTestNode{ - int64(8001), Position{13, 1}, - }, - queryTestNode{ - int64(8002), Position{13, 1}, - }, - queryTestNode{ - int64(5000), Position{14, 1}, - }, - }) - - assertQueryPositions(t, string(buff), - "$..[?(string)]", - []interface{}{ - queryTestNode{ - "TOML Example", Position{3, 1}, - }, - queryTestNode{ - "Tom Preston-Werner", Position{6, 1}, - }, - queryTestNode{ - "GitHub", Position{7, 1}, - }, - queryTestNode{ - "GitHub Cofounder & CEO\nLikes tater tots and beer.", - Position{8, 1}, - }, - queryTestNode{ - "192.168.1.1", Position{12, 1}, - }, - queryTestNode{ - "10.0.0.1", Position{21, 3}, - }, - queryTestNode{ - "eqdc10", Position{22, 3}, - }, - queryTestNode{ - "10.0.0.2", Position{25, 3}, - }, - queryTestNode{ - "eqdc10", Position{26, 3}, - }, - }) - - assertQueryPositions(t, string(buff), - "$..[?(float)]", - []interface{}{ - // no float values in document - }) - - tv, _ := time.Parse(time.RFC3339, "1979-05-27T07:32:00Z") - assertQueryPositions(t, string(buff), - "$..[?(tree)]", - []interface{}{ - queryTestNode{ - map[string]interface{}{ - "name": "Tom Preston-Werner", - "organization": "GitHub", - "bio": "GitHub Cofounder & CEO\nLikes tater tots and beer.", - "dob": tv, - }, Position{5, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "server": "192.168.1.1", - "ports": []interface{}{int64(8001), int64(8001), int64(8002)}, - "connection_max": int64(5000), - "enabled": true, - }, Position{11, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "alpha": map[string]interface{}{ - "ip": "10.0.0.1", - "dc": "eqdc10", - }, - "beta": map[string]interface{}{ - "ip": "10.0.0.2", - "dc": "eqdc10", - }, - }, Position{17, 1}, - }, - queryTestNode{ - map[string]interface{}{ - "ip": "10.0.0.1", - "dc": "eqdc10", - }, Position{20, 3}, - }, - queryTestNode{ - map[string]interface{}{ - "ip": "10.0.0.2", - "dc": "eqdc10", - }, Position{24, 3}, - }, - queryTestNode{ - map[string]interface{}{ - "data": []interface{}{ - []interface{}{"gamma", "delta"}, - []interface{}{int64(1), int64(2)}, - }, - }, Position{28, 1}, - }, - }) - - assertQueryPositions(t, string(buff), - "$..[?(time)]", - []interface{}{ - queryTestNode{ - tv, Position{9, 1}, - }, - }) - - assertQueryPositions(t, string(buff), - "$..[?(bool)]", - []interface{}{ - queryTestNode{ - true, Position{15, 1}, - }, - }) -} diff --git a/vendor/github.com/pelletier/go-toml/test.sh b/vendor/github.com/pelletier/go-toml/test.sh index 436d2fb6ca..91a889670f 100755 --- a/vendor/github.com/pelletier/go-toml/test.sh +++ b/vendor/github.com/pelletier/go-toml/test.sh @@ -19,11 +19,16 @@ function git_clone() { popd } +# Remove potential previous runs +rm -rf src test_program_bin toml-test + # Run go vet go vet ./... go get github.com/pelletier/go-buffruneio go get github.com/davecgh/go-spew/spew +go get gopkg.in/yaml.v2 +go get github.com/BurntSushi/toml # get code for BurntSushi TOML validation # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize) @@ -36,13 +41,16 @@ go build -o toml-test github.com/BurntSushi/toml-test # vendorize the current lib for testing # NOTE: this basically mocks an install without having to go back out to github for code mkdir -p src/github.com/pelletier/go-toml/cmd +mkdir -p src/github.com/pelletier/go-toml/query cp *.go *.toml src/github.com/pelletier/go-toml cp -R cmd/* src/github.com/pelletier/go-toml/cmd +cp -R query/* src/github.com/pelletier/go-toml/query go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go # Run basic unit tests -go test github.com/pelletier/go-toml -v -covermode=count -coverprofile=coverage.out +go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out go test github.com/pelletier/go-toml/cmd/tomljson +go test github.com/pelletier/go-toml/query # run the entire BurntSushi test suite if [[ $# -eq 0 ]] ; then diff --git a/vendor/github.com/pelletier/go-toml/toml.go b/vendor/github.com/pelletier/go-toml/toml.go index 1ba56a1c14..c3e324374a 100644 --- a/vendor/github.com/pelletier/go-toml/toml.go +++ b/vendor/github.com/pelletier/go-toml/toml.go @@ -4,40 +4,50 @@ import ( "errors" "fmt" "io" + "io/ioutil" "os" "runtime" "strings" ) type tomlValue struct { - value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list - position Position + value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list + comment string + commented bool + position Position } -// TomlTree is the result of the parsing of a TOML file. -type TomlTree struct { - values map[string]interface{} // string -> *tomlValue, *TomlTree, []*TomlTree - position Position +// Tree is the result of the parsing of a TOML file. +type Tree struct { + values map[string]interface{} // string -> *tomlValue, *Tree, []*Tree + comment string + commented bool + position Position } -func newTomlTree() *TomlTree { - return &TomlTree{ +func newTree() *Tree { + return &Tree{ values: make(map[string]interface{}), position: Position{}, } } -// TreeFromMap initializes a new TomlTree object using the given map. -func TreeFromMap(m map[string]interface{}) (*TomlTree, error) { +// TreeFromMap initializes a new Tree object using the given map. +func TreeFromMap(m map[string]interface{}) (*Tree, error) { result, err := toTree(m) if err != nil { return nil, err } - return result.(*TomlTree), nil + return result.(*Tree), nil +} + +// Position returns the position of the tree. +func (t *Tree) Position() Position { + return t.position } // Has returns a boolean indicating if the given key exists. -func (t *TomlTree) Has(key string) bool { +func (t *Tree) Has(key string) bool { if key == "" { return false } @@ -45,25 +55,26 @@ func (t *TomlTree) Has(key string) bool { } // HasPath returns true if the given path of keys exists, false otherwise. -func (t *TomlTree) HasPath(keys []string) bool { +func (t *Tree) HasPath(keys []string) bool { return t.GetPath(keys) != nil } -// Keys returns the keys of the toplevel tree. -// Warning: this is a costly operation. -func (t *TomlTree) Keys() []string { - var keys []string +// Keys returns the keys of the toplevel tree (does not recurse). +func (t *Tree) Keys() []string { + keys := make([]string, len(t.values)) + i := 0 for k := range t.values { - keys = append(keys, k) + keys[i] = k + i++ } return keys } -// Get the value at key in the TomlTree. +// Get the value at key in the Tree. // Key is a dot-separated path (e.g. a.b.c). // Returns nil if the path does not exist in the tree. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) Get(key string) interface{} { +func (t *Tree) Get(key string) interface{} { if key == "" { return t } @@ -76,7 +87,7 @@ func (t *TomlTree) Get(key string) interface{} { // GetPath returns the element in the tree indicated by 'keys'. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) GetPath(keys []string) interface{} { +func (t *Tree) GetPath(keys []string) interface{} { if len(keys) == 0 { return t } @@ -87,9 +98,9 @@ func (t *TomlTree) GetPath(keys []string) interface{} { return nil } switch node := value.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return nil @@ -109,7 +120,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} { } // GetPosition returns the position of the given key. -func (t *TomlTree) GetPosition(key string) Position { +func (t *Tree) GetPosition(key string) Position { if key == "" { return t.position } @@ -118,7 +129,7 @@ func (t *TomlTree) GetPosition(key string) Position { // GetPositionPath returns the element in the tree indicated by 'keys'. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) GetPositionPath(keys []string) Position { +func (t *Tree) GetPositionPath(keys []string) Position { if len(keys) == 0 { return t.position } @@ -129,9 +140,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { return Position{0, 0} } switch node := value.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return Position{0, 0} @@ -145,9 +156,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { switch node := subtree.values[keys[len(keys)-1]].(type) { case *tomlValue: return node.position - case *TomlTree: + case *Tree: return node.position - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return Position{0, 0} @@ -159,7 +170,7 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { } // GetDefault works like Get but with a default value -func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { +func (t *Tree) GetDefault(key string, def interface{}) interface{} { val := t.Get(key) if val == nil { return def @@ -169,30 +180,30 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { // Set an element in the tree. // Key is a dot-separated path (e.g. a.b.c). -// Creates all necessary intermediates trees, if needed. -func (t *TomlTree) Set(key string, value interface{}) { - t.SetPath(strings.Split(key, "."), value) +// Creates all necessary intermediate trees, if needed. +func (t *Tree) Set(key string, comment string, commented bool, value interface{}) { + t.SetPath(strings.Split(key, "."), comment, commented, value) } // SetPath sets an element in the tree. // Keys is an array of path elements (e.g. {"a","b","c"}). -// Creates all necessary intermediates trees, if needed. -func (t *TomlTree) SetPath(keys []string, value interface{}) { +// Creates all necessary intermediate trees, if needed. +func (t *Tree) SetPath(keys []string, comment string, commented bool, value interface{}) { subtree := t for _, intermediateKey := range keys[:len(keys)-1] { nextTree, exists := subtree.values[intermediateKey] if !exists { - nextTree = newTomlTree() + nextTree = newTree() subtree.values[intermediateKey] = nextTree // add new element here } switch node := nextTree.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { // create element if it does not exist - subtree.values[intermediateKey] = append(node, newTomlTree()) + subtree.values[intermediateKey] = append(node, newTree()) } subtree = node[len(node)-1] } @@ -201,14 +212,18 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) { var toInsert interface{} switch value.(type) { - case *TomlTree: + case *Tree: + tt := value.(*Tree) + tt.comment = comment toInsert = value - case []*TomlTree: + case []*Tree: toInsert = value case *tomlValue: - toInsert = value + tt := value.(*tomlValue) + tt.comment = comment + toInsert = tt default: - toInsert = &tomlValue{value: value} + toInsert = &tomlValue{value: value, comment: comment, commented: commented} } subtree.values[keys[len(keys)-1]] = toInsert @@ -221,21 +236,21 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) { // and tree[a][b][c] // // Returns nil on success, error object on failure -func (t *TomlTree) createSubTree(keys []string, pos Position) error { +func (t *Tree) createSubTree(keys []string, pos Position) error { subtree := t for _, intermediateKey := range keys { nextTree, exists := subtree.values[intermediateKey] if !exists { - tree := newTomlTree() + tree := newTree() tree.position = pos subtree.values[intermediateKey] = tree nextTree = tree } switch node := nextTree.(type) { - case []*TomlTree: + case []*Tree: subtree = node[len(node)-1] - case *TomlTree: + case *Tree: subtree = node default: return fmt.Errorf("unknown type for path %s (%s): %T (%#v)", @@ -245,17 +260,8 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error { return nil } -// Query compiles and executes a query on a tree and returns the query result. -func (t *TomlTree) Query(query string) (*QueryResult, error) { - q, err := CompileQuery(query) - if err != nil { - return nil, err - } - return q.Execute(t), nil -} - -// LoadReader creates a TomlTree from any io.Reader. -func LoadReader(reader io.Reader) (tree *TomlTree, err error) { +// LoadBytes creates a Tree from a []byte. +func LoadBytes(b []byte) (tree *Tree, err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { @@ -264,17 +270,27 @@ func LoadReader(reader io.Reader) (tree *TomlTree, err error) { err = errors.New(r.(string)) } }() - tree = parseToml(lexToml(reader)) + tree = parseToml(lexToml(b)) + return +} + +// LoadReader creates a Tree from any io.Reader. +func LoadReader(reader io.Reader) (tree *Tree, err error) { + inputBytes, err := ioutil.ReadAll(reader) + if err != nil { + return + } + tree, err = LoadBytes(inputBytes) return } -// Load creates a TomlTree from a string. -func Load(content string) (tree *TomlTree, err error) { - return LoadReader(strings.NewReader(content)) +// Load creates a Tree from a string. +func Load(content string) (tree *Tree, err error) { + return LoadBytes([]byte(content)) } -// LoadFile creates a TomlTree from a file. -func LoadFile(path string) (tree *TomlTree, err error) { +// LoadFile creates a Tree from a file. +func LoadFile(path string) (tree *Tree, err error) { file, err := os.Open(path) if err != nil { return nil, err diff --git a/vendor/github.com/pelletier/go-toml/toml_test.go b/vendor/github.com/pelletier/go-toml/toml_test.go index 7c7f9eff95..ab9c242535 100644 --- a/vendor/github.com/pelletier/go-toml/toml_test.go +++ b/vendor/github.com/pelletier/go-toml/toml_test.go @@ -70,12 +70,12 @@ func TestTomlHasPath(t *testing.T) { } func TestTomlGetPath(t *testing.T) { - node := newTomlTree() + node := newTree() //TODO: set other node data for idx, item := range []struct { Path []string - Expected *TomlTree + Expected *Tree }{ { // empty path test []string{}, @@ -94,31 +94,6 @@ func TestTomlGetPath(t *testing.T) { } } -func TestTomlQuery(t *testing.T) { - tree, err := Load("[foo.bar]\na=1\nb=2\n[baz.foo]\na=3\nb=4\n[gorf.foo]\na=5\nb=6") - if err != nil { - t.Error(err) - return - } - result, err := tree.Query("$.foo.bar") - if err != nil { - t.Error(err) - return - } - values := result.Values() - if len(values) != 1 { - t.Errorf("Expected resultset of 1, got %d instead: %v", len(values), values) - } - - if tt, ok := values[0].(*TomlTree); !ok { - t.Errorf("Expected type of TomlTree: %T", values[0]) - } else if tt.Get("a") != int64(1) { - t.Errorf("Expected 'a' with a value 1: %v", tt.Get("a")) - } else if tt.Get("b") != int64(2) { - t.Errorf("Expected 'b' with a value 2: %v", tt.Get("b")) - } -} - func TestTomlFromMap(t *testing.T) { simpleMap := map[string]interface{}{"hello": 42} tree, err := TreeFromMap(simpleMap) diff --git a/vendor/github.com/pelletier/go-toml/tomltree_create.go b/vendor/github.com/pelletier/go-toml/tomltree_create.go index c6054f358b..79610e9b34 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_create.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_create.go @@ -6,10 +6,7 @@ import ( "time" ) -// supported values: -// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 - -var kindToTypeMapping = map[reflect.Kind]reflect.Type{ +var kindToType = [reflect.String + 1]reflect.Type{ reflect.Bool: reflect.TypeOf(true), reflect.String: reflect.TypeOf(""), reflect.Float32: reflect.TypeOf(float64(1)), @@ -26,6 +23,16 @@ var kindToTypeMapping = map[reflect.Kind]reflect.Type{ reflect.Uint64: reflect.TypeOf(uint64(1)), } +// typeFor returns a reflect.Type for a reflect.Kind, or nil if none is found. +// supported values: +// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 +func typeFor(k reflect.Kind) reflect.Type { + if k > 0 && int(k) < len(kindToType) { + return kindToType[k] + } + return nil +} + func simpleValueCoercion(object interface{}) (interface{}, error) { switch original := object.(type) { case string, bool, int64, uint64, float64, time.Time: @@ -51,7 +58,7 @@ func simpleValueCoercion(object interface{}) (interface{}, error) { case fmt.Stringer: return original.String(), nil default: - return nil, fmt.Errorf("cannot convert type %T to TomlTree", object) + return nil, fmt.Errorf("cannot convert type %T to Tree", object) } } @@ -59,7 +66,7 @@ func sliceToTree(object interface{}) (interface{}, error) { // arrays are a bit tricky, since they can represent either a // collection of simple values, which is represented by one // *tomlValue, or an array of tables, which is represented by an - // array of *TomlTree. + // array of *Tree. // holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice value := reflect.ValueOf(object) @@ -70,19 +77,19 @@ func sliceToTree(object interface{}) (interface{}, error) { } if insideType.Kind() == reflect.Map { // this is considered as an array of tables - tablesArray := make([]*TomlTree, 0, length) + tablesArray := make([]*Tree, 0, length) for i := 0; i < length; i++ { table := value.Index(i) tree, err := toTree(table.Interface()) if err != nil { return nil, err } - tablesArray = append(tablesArray, tree.(*TomlTree)) + tablesArray = append(tablesArray, tree.(*Tree)) } return tablesArray, nil } - sliceType := kindToTypeMapping[insideType.Kind()] + sliceType := typeFor(insideType.Kind()) if sliceType == nil { sliceType = insideType } @@ -97,7 +104,7 @@ func sliceToTree(object interface{}) (interface{}, error) { } arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue)) } - return &tomlValue{arrayValue.Interface(), Position{}}, nil + return &tomlValue{value: arrayValue.Interface(), position: Position{}}, nil } func toTree(object interface{}) (interface{}, error) { @@ -120,7 +127,7 @@ func toTree(object interface{}) (interface{}, error) { } values[key.String()] = newValue } - return &TomlTree{values, Position{}}, nil + return &Tree{values: values, position: Position{}}, nil } if value.Kind() == reflect.Array || value.Kind() == reflect.Slice { @@ -131,5 +138,5 @@ func toTree(object interface{}) (interface{}, error) { if err != nil { return nil, err } - return &tomlValue{simpleValue, Position{}}, nil + return &tomlValue{value: simpleValue, position: Position{}}, nil } diff --git a/vendor/github.com/pelletier/go-toml/tomltree_create_test.go b/vendor/github.com/pelletier/go-toml/tomltree_create_test.go index 6c1496835e..1ca108a524 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_create_test.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_create_test.go @@ -1,9 +1,9 @@ package toml import ( + "strconv" "testing" "time" - "strconv" ) type customString string @@ -16,11 +16,11 @@ func (s stringer) String() string { func validate(t *testing.T, path string, object interface{}) { switch o := object.(type) { - case *TomlTree: + case *Tree: for key, tree := range o.values { validate(t, path+"."+key, tree) } - case []*TomlTree: + case []*Tree: for index, tree := range o { validate(t, path+"."+strconv.Itoa(index), tree) } @@ -37,11 +37,11 @@ func validate(t *testing.T, path string, object interface{}) { t.Logf("validation ok %s as %T", path, object) } -func validateTree(t *testing.T, tree *TomlTree) { +func validateTree(t *testing.T, tree *Tree) { validate(t, "", tree) } -func TestTomlTreeCreateToTree(t *testing.T) { +func TestTreeCreateToTree(t *testing.T) { data := map[string]interface{}{ "a_string": "bar", "an_int": 42, @@ -72,15 +72,15 @@ func TestTomlTreeCreateToTree(t *testing.T) { validateTree(t, tree) } -func TestTomlTreeCreateToTreeInvalidLeafType(t *testing.T) { +func TestTreeCreateToTreeInvalidLeafType(t *testing.T) { _, err := TreeFromMap(map[string]interface{}{"foo": t}) - expected := "cannot convert type *testing.T to TomlTree" + expected := "cannot convert type *testing.T to Tree" if err.Error() != expected { t.Fatalf("expected error %s, got %s", expected, err.Error()) } } -func TestTomlTreeCreateToTreeInvalidMapKeyType(t *testing.T) { +func TestTreeCreateToTreeInvalidMapKeyType(t *testing.T) { _, err := TreeFromMap(map[string]interface{}{"foo": map[int]interface{}{2: 1}}) expected := "map key needs to be a string, not int (int)" if err.Error() != expected { @@ -88,17 +88,17 @@ func TestTomlTreeCreateToTreeInvalidMapKeyType(t *testing.T) { } } -func TestTomlTreeCreateToTreeInvalidArrayMemberType(t *testing.T) { +func TestTreeCreateToTreeInvalidArrayMemberType(t *testing.T) { _, err := TreeFromMap(map[string]interface{}{"foo": []*testing.T{t}}) - expected := "cannot convert type *testing.T to TomlTree" + expected := "cannot convert type *testing.T to Tree" if err.Error() != expected { t.Fatalf("expected error %s, got %s", expected, err.Error()) } } -func TestTomlTreeCreateToTreeInvalidTableGroupType(t *testing.T) { +func TestTreeCreateToTreeInvalidTableGroupType(t *testing.T) { _, err := TreeFromMap(map[string]interface{}{"foo": []map[string]interface{}{map[string]interface{}{"hello": t}}}) - expected := "cannot convert type *testing.T to TomlTree" + expected := "cannot convert type *testing.T to Tree" if err.Error() != expected { t.Fatalf("expected error %s, got %s", expected, err.Error()) } diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write.go b/vendor/github.com/pelletier/go-toml/tomltree_write.go index 6a7fa17458..f5ef124f0b 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_write.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_write.go @@ -4,57 +4,64 @@ import ( "bytes" "fmt" "io" + "math" + "reflect" "sort" "strconv" "strings" "time" - "reflect" ) // encodes a string to a TOML-compliant string value func encodeTomlString(value string) string { - result := "" + var b bytes.Buffer + for _, rr := range value { switch rr { case '\b': - result += "\\b" + b.WriteString(`\b`) case '\t': - result += "\\t" + b.WriteString(`\t`) case '\n': - result += "\\n" + b.WriteString(`\n`) case '\f': - result += "\\f" + b.WriteString(`\f`) case '\r': - result += "\\r" + b.WriteString(`\r`) case '"': - result += "\\\"" + b.WriteString(`\"`) case '\\': - result += "\\\\" + b.WriteString(`\\`) default: intRr := uint16(rr) if intRr < 0x001F { - result += fmt.Sprintf("\\u%0.4X", intRr) + b.WriteString(fmt.Sprintf("\\u%0.4X", intRr)) } else { - result += string(rr) + b.WriteRune(rr) } } } - return result + return b.String() } -func tomlValueStringRepresentation(v interface{}) (string, error) { +func tomlValueStringRepresentation(v interface{}, indent string, arraysOneElementPerLine bool) (string, error) { switch value := v.(type) { case uint64: return strconv.FormatUint(value, 10), nil case int64: return strconv.FormatInt(value, 10), nil case float64: + // Ensure a round float does contain a decimal point. Otherwise feeding + // the output back to the parser would convert to an integer. + if math.Trunc(value) == value { + return strconv.FormatFloat(value, 'f', 1, 32), nil + } return strconv.FormatFloat(value, 'f', -1, 32), nil case string: return "\"" + encodeTomlString(value) + "\"", nil case []byte: b, _ := v.([]byte) - return tomlValueStringRepresentation(string(b)) + return tomlValueStringRepresentation(string(b), indent, arraysOneElementPerLine) case bool: if value { return "true", nil @@ -69,28 +76,47 @@ func tomlValueStringRepresentation(v interface{}) (string, error) { rv := reflect.ValueOf(v) if rv.Kind() == reflect.Slice { - values := []string{} + var values []string for i := 0; i < rv.Len(); i++ { item := rv.Index(i).Interface() - itemRepr, err := tomlValueStringRepresentation(item) + itemRepr, err := tomlValueStringRepresentation(item, indent, arraysOneElementPerLine) if err != nil { return "", err } values = append(values, itemRepr) } + if arraysOneElementPerLine && len(values) > 1 { + stringBuffer := bytes.Buffer{} + valueIndent := indent + ` ` // TODO: move that to a shared encoder state + + stringBuffer.WriteString("[\n") + + for i, value := range values { + stringBuffer.WriteString(valueIndent) + stringBuffer.WriteString(value) + if i != len(values)-1 { + stringBuffer.WriteString(`,`) + } + stringBuffer.WriteString("\n") + } + + stringBuffer.WriteString(indent + "]") + + return stringBuffer.String(), nil + } return "[" + strings.Join(values, ",") + "]", nil } return "", fmt.Errorf("unsupported value type %T: %v", v, v) } -func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) { +func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64, arraysOneElementPerLine bool) (int64, error) { simpleValuesKeys := make([]string, 0) complexValuesKeys := make([]string, 0) for k := range t.values { v := t.values[k] switch v.(type) { - case *TomlTree, []*TomlTree: + case *Tree, []*Tree: complexValuesKeys = append(complexValuesKeys, k) default: simpleValuesKeys = append(simpleValuesKeys, k) @@ -106,13 +132,29 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) } - repr, err := tomlValueStringRepresentation(v.value) + repr, err := tomlValueStringRepresentation(v.value, indent, arraysOneElementPerLine) if err != nil { return bytesCount, err } - kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr) - writtenBytesCount, err := w.Write([]byte(kvRepr)) + if v.comment != "" { + comment := strings.Replace(v.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment, "\n") + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + + var commented string + if v.commented { + commented = "# " + } + writtenBytesCount, err := writeStrings(w, indent, commented, k, " = ", repr, "\n") bytesCount += int64(writtenBytesCount) if err != nil { return bytesCount, err @@ -126,34 +168,50 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 if keyspace != "" { combinedKey = keyspace + "." + combinedKey } + var commented string + if t.commented { + commented = "# " + } switch node := v.(type) { // node has to be of those two types given how keys are sorted above - case *TomlTree: - tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - writtenBytesCount, err := w.Write([]byte(tableName)) + case *Tree: + tv, ok := t.values[k].(*Tree) + if !ok { + return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) + } + if tv.comment != "" { + comment := strings.Replace(tv.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment) + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[", combinedKey, "]\n") bytesCount += int64(writtenBytesCount) if err != nil { return bytesCount, err } - bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount) + bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) if err != nil { return bytesCount, err } - case []*TomlTree: + case []*Tree: for _, subTree := range node { - if len(subTree.values) > 0 { - tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey) - writtenBytesCount, err := w.Write([]byte(tableArrayName)) - bytesCount += int64(writtenBytesCount) - if err != nil { - return bytesCount, err - } - - bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount) - if err != nil { - return bytesCount, err - } + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[[", combinedKey, "]]\n") + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + + bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) + if err != nil { + return bytesCount, err } } } @@ -162,16 +220,28 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 return bytesCount, nil } -// WriteTo encode the TomlTree as Toml and writes it to the writer w. +func writeStrings(w io.Writer, s ...string) (int, error) { + var n int + for i := range s { + b, err := io.WriteString(w, s[i]) + n += b + if err != nil { + return n, err + } + } + return n, nil +} + +// WriteTo encode the Tree as Toml and writes it to the writer w. // Returns the number of bytes written in case of success, or an error if anything happened. -func (t *TomlTree) WriteTo(w io.Writer) (int64, error) { - return t.writeTo(w, "", "", 0) +func (t *Tree) WriteTo(w io.Writer) (int64, error) { + return t.writeTo(w, "", "", 0, false) } // ToTomlString generates a human-readable representation of the current tree. // Output spans multiple lines, and is suitable for ingest by a TOML parser. // If the conversion cannot be performed, ToString returns a non-nil error. -func (t *TomlTree) ToTomlString() (string, error) { +func (t *Tree) ToTomlString() (string, error) { var buf bytes.Buffer _, err := t.WriteTo(&buf) if err != nil { @@ -182,32 +252,34 @@ func (t *TomlTree) ToTomlString() (string, error) { // String generates a human-readable representation of the current tree. // Alias of ToString. Present to implement the fmt.Stringer interface. -func (t *TomlTree) String() string { +func (t *Tree) String() string { result, _ := t.ToTomlString() return result } // ToMap recursively generates a representation of the tree using Go built-in structures. // The following types are used: -// * uint64 -// * int64 -// * bool -// * string -// * time.Time -// * map[string]interface{} (where interface{} is any of this list) -// * []interface{} (where interface{} is any of this list) -func (t *TomlTree) ToMap() map[string]interface{} { +// +// * bool +// * float64 +// * int64 +// * string +// * uint64 +// * time.Time +// * map[string]interface{} (where interface{} is any of this list) +// * []interface{} (where interface{} is any of this list) +func (t *Tree) ToMap() map[string]interface{} { result := map[string]interface{}{} for k, v := range t.values { switch node := v.(type) { - case []*TomlTree: + case []*Tree: var array []interface{} for _, item := range node { array = append(array, item.ToMap()) } result[k] = array - case *TomlTree: + case *Tree: result[k] = node.ToMap() case *tomlValue: result[k] = node.value diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write_test.go b/vendor/github.com/pelletier/go-toml/tomltree_write_test.go index b5ad8db7c0..5ea59bc1a7 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_write_test.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_write_test.go @@ -16,31 +16,55 @@ type failingWriter struct { buffer bytes.Buffer } -func (f failingWriter) Write(p []byte) (n int, err error) { +func (f *failingWriter) Write(p []byte) (n int, err error) { count := len(p) - toWrite := f.failAt - count + f.written + toWrite := f.failAt - (count + f.written) if toWrite < 0 { toWrite = 0 } if toWrite > count { f.written += count - f.buffer.WriteString(string(p)) + f.buffer.Write(p) return count, nil } - f.buffer.WriteString(string(p[:toWrite])) + f.buffer.Write(p[:toWrite]) f.written = f.failAt - return f.written, fmt.Errorf("failingWriter failed after writting %d bytes", f.written) + return toWrite, fmt.Errorf("failingWriter failed after writing %d bytes", f.written) } func assertErrorString(t *testing.T, expected string, err error) { expectedErr := errors.New(expected) - if err.Error() != expectedErr.Error() { + if err == nil || err.Error() != expectedErr.Error() { t.Errorf("expecting error %s, but got %s instead", expected, err) } } -func TestTomlTreeWriteToTomlString(t *testing.T) { +func TestTreeWriteToEmptyTable(t *testing.T) { + doc := `[[empty-tables]] +[[empty-tables]]` + + toml, err := Load(doc) + if err != nil { + t.Fatal("Unexpected Load error:", err) + } + tomlString, err := toml.ToTomlString() + if err != nil { + t.Fatal("Unexpected ToTomlString error:", err) + } + + expected := ` +[[empty-tables]] + +[[empty-tables]] +` + + if tomlString != expected { + t.Fatalf("Expected:\n%s\nGot:\n%s", expected, tomlString) + } +} + +func TestTreeWriteToTomlString(t *testing.T) { toml, err := Load(`name = { first = "Tom", last = "Preston-Werner" } points = { x = 1, y = 2 }`) @@ -63,7 +87,7 @@ points = { x = 1, y = 2 }`) }) } -func TestTomlTreeWriteToTomlStringSimple(t *testing.T) { +func TestTreeWriteToTomlStringSimple(t *testing.T) { tree, err := Load("[foo]\n\n[[foo.bar]]\na = 42\n\n[[foo.bar]]\na = 69\n") if err != nil { t.Errorf("Test failed to parse: %v", err) @@ -79,7 +103,7 @@ func TestTomlTreeWriteToTomlStringSimple(t *testing.T) { } } -func TestTomlTreeWriteToTomlStringKeysOrders(t *testing.T) { +func TestTreeWriteToTomlStringKeysOrders(t *testing.T) { for i := 0; i < 100; i++ { tree, _ := Load(` foobar = true @@ -119,7 +143,7 @@ func testMaps(t *testing.T, actual, expected map[string]interface{}) { } } -func TestTomlTreeWriteToMapSimple(t *testing.T) { +func TestTreeWriteToMapSimple(t *testing.T) { tree, _ := Load("a = 42\nb = 17") expected := map[string]interface{}{ @@ -130,58 +154,58 @@ func TestTomlTreeWriteToMapSimple(t *testing.T) { testMaps(t, tree.ToMap(), expected) } -func TestTomlTreeWriteToInvalidTreeSimpleValue(t *testing.T) { - tree := TomlTree{values: map[string]interface{}{"foo": int8(1)}} +func TestTreeWriteToInvalidTreeSimpleValue(t *testing.T) { + tree := Tree{values: map[string]interface{}{"foo": int8(1)}} _, err := tree.ToTomlString() assertErrorString(t, "invalid value type at foo: int8", err) } -func TestTomlTreeWriteToInvalidTreeTomlValue(t *testing.T) { - tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{int8(1), Position{}}}} +func TestTreeWriteToInvalidTreeTomlValue(t *testing.T) { + tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}} _, err := tree.ToTomlString() assertErrorString(t, "unsupported value type int8: 1", err) } -func TestTomlTreeWriteToInvalidTreeTomlValueArray(t *testing.T) { - tree := TomlTree{values: map[string]interface{}{"foo": &tomlValue{[]interface{}{int8(1)}, Position{}}}} +func TestTreeWriteToInvalidTreeTomlValueArray(t *testing.T) { + tree := Tree{values: map[string]interface{}{"foo": &tomlValue{value: int8(1), comment: "", position: Position{}}}} _, err := tree.ToTomlString() assertErrorString(t, "unsupported value type int8: 1", err) } -func TestTomlTreeWriteToFailingWriterInSimpleValue(t *testing.T) { +func TestTreeWriteToFailingWriterInSimpleValue(t *testing.T) { toml, _ := Load(`a = 2`) writer := failingWriter{failAt: 0, written: 0} - _, err := toml.WriteTo(writer) - assertErrorString(t, "failingWriter failed after writting 0 bytes", err) + _, err := toml.WriteTo(&writer) + assertErrorString(t, "failingWriter failed after writing 0 bytes", err) } -func TestTomlTreeWriteToFailingWriterInTable(t *testing.T) { +func TestTreeWriteToFailingWriterInTable(t *testing.T) { toml, _ := Load(` [b] a = 2`) writer := failingWriter{failAt: 2, written: 0} - _, err := toml.WriteTo(writer) - assertErrorString(t, "failingWriter failed after writting 2 bytes", err) + _, err := toml.WriteTo(&writer) + assertErrorString(t, "failingWriter failed after writing 2 bytes", err) writer = failingWriter{failAt: 13, written: 0} - _, err = toml.WriteTo(writer) - assertErrorString(t, "failingWriter failed after writting 13 bytes", err) + _, err = toml.WriteTo(&writer) + assertErrorString(t, "failingWriter failed after writing 13 bytes", err) } -func TestTomlTreeWriteToFailingWriterInArray(t *testing.T) { +func TestTreeWriteToFailingWriterInArray(t *testing.T) { toml, _ := Load(` [[b]] a = 2`) writer := failingWriter{failAt: 2, written: 0} - _, err := toml.WriteTo(writer) - assertErrorString(t, "failingWriter failed after writting 2 bytes", err) + _, err := toml.WriteTo(&writer) + assertErrorString(t, "failingWriter failed after writing 2 bytes", err) writer = failingWriter{failAt: 15, written: 0} - _, err = toml.WriteTo(writer) - assertErrorString(t, "failingWriter failed after writting 15 bytes", err) + _, err = toml.WriteTo(&writer) + assertErrorString(t, "failingWriter failed after writing 15 bytes", err) } -func TestTomlTreeWriteToMapExampleFile(t *testing.T) { +func TestTreeWriteToMapExampleFile(t *testing.T) { tree, _ := LoadFile("example.toml") expected := map[string]interface{}{ "title": "TOML Example", @@ -217,7 +241,7 @@ func TestTomlTreeWriteToMapExampleFile(t *testing.T) { testMaps(t, tree.ToMap(), expected) } -func TestTomlTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) { +func TestTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) { tree, _ := Load(` [[menu.main]] a = "menu 1" @@ -238,7 +262,7 @@ func TestTomlTreeWriteToMapWithTablesInMultipleChunks(t *testing.T) { testMaps(t, treeMap, expected) } -func TestTomlTreeWriteToMapWithArrayOfInlineTables(t *testing.T) { +func TestTreeWriteToMapWithArrayOfInlineTables(t *testing.T) { tree, _ := Load(` [params] language_tabs = [ @@ -269,3 +293,66 @@ func TestTomlTreeWriteToMapWithArrayOfInlineTables(t *testing.T) { treeMap := tree.ToMap() testMaps(t, treeMap, expected) } + +func TestTreeWriteToFloat(t *testing.T) { + tree, err := Load(`a = 3.0`) + if err != nil { + t.Fatal(err) + } + str, err := tree.ToTomlString() + if err != nil { + t.Fatal(err) + } + expected := `a = 3.0` + if strings.TrimSpace(str) != strings.TrimSpace(expected) { + t.Fatalf("Expected:\n%s\nGot:\n%s", expected, str) + } +} + +func BenchmarkTreeToTomlString(b *testing.B) { + toml, err := Load(sampleHard) + if err != nil { + b.Fatal("Unexpected error:", err) + } + + for i := 0; i < b.N; i++ { + _, err := toml.ToTomlString() + if err != nil { + b.Fatal(err) + } + } +} + +var sampleHard = `# Test file for TOML +# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate +# This part you'll really hate + +[the] +test_string = "You'll hate me after this - #" # " Annoying, isn't it? + + [the.hard] + test_array = [ "] ", " # "] # ] There you go, parse this! + test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ] + # You didn't think it'd as easy as chucking out the last #, did you? + another_test_string = " Same thing, but with a string #" + harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too" + # Things will get harder + + [the.hard."bit#"] + "what?" = "You don't think some user won't do that?" + multi_line_array = [ + "]", + # ] Oh yes I did + ] + +# Each of the following keygroups/key value pairs should produce an error. Uncomment to them to test + +#[error] if you didn't catch this, your parser is broken +#string = "Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment" like this +#array = [ +# "This might most likely happen in multiline arrays", +# Like here, +# "or here, +# and here" +# ] End of array comment, forgot the # +#number = 3.14 pi <--again forgot the # ` From d6dde6170b897cf9d0185575a4fd55b5625c72ee Mon Sep 17 00:00:00 2001 From: sam boyer Date: Mon, 18 Dec 2017 09:18:13 -0500 Subject: [PATCH 2/3] dep: Use new line-per-item behaviors from TOML lib --- .../ensure/add/all-new-double/final/Gopkg.lock | 5 ++++- .../harness_tests/init/glide/case1/final/Gopkg.toml | 5 ++++- .../harness_tests/init/govendor/case1/final/Gopkg.toml | 5 ++++- lock.go | 6 ++++-- manifest.go | 6 ++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/dep/testdata/harness_tests/ensure/add/all-new-double/final/Gopkg.lock b/cmd/dep/testdata/harness_tests/ensure/add/all-new-double/final/Gopkg.lock index d2b0ea3287..8481da4451 100644 --- a/cmd/dep/testdata/harness_tests/ensure/add/all-new-double/final/Gopkg.lock +++ b/cmd/dep/testdata/harness_tests/ensure/add/all-new-double/final/Gopkg.lock @@ -10,7 +10,10 @@ [[projects]] branch = "master" name = "github.com/sdboyer/deptesttres" - packages = [".","subp"] + packages = [ + ".", + "subp" + ] revision = "54aaeb0023e1f3dcf5f98f31dd8c565457945a12" [solve-meta] diff --git a/cmd/dep/testdata/harness_tests/init/glide/case1/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/init/glide/case1/final/Gopkg.toml index c0dce36f71..0cc8a698e8 100644 --- a/cmd/dep/testdata/harness_tests/init/glide/case1/final/Gopkg.toml +++ b/cmd/dep/testdata/harness_tests/init/glide/case1/final/Gopkg.toml @@ -1,4 +1,7 @@ -ignored = ["github.com/golang/notexist/samples","github.com/sdboyer/dep-test"] +ignored = [ + "github.com/golang/notexist/samples", + "github.com/sdboyer/dep-test" +] [[constraint]] name = "github.com/sdboyer/deptestdos" diff --git a/cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml index adef29b97a..55033c0257 100644 --- a/cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml +++ b/cmd/dep/testdata/harness_tests/init/govendor/case1/final/Gopkg.toml @@ -1,4 +1,7 @@ -ignored = ["github.com/golang/notexist/samples*","github.com/sdboyer/dep-test*"] +ignored = [ + "github.com/golang/notexist/samples*", + "github.com/sdboyer/dep-test*" +] [[constraint]] name = "github.com/sdboyer/deptestdos" diff --git a/lock.go b/lock.go index 25d74d426c..3f3f563c42 100644 --- a/lock.go +++ b/lock.go @@ -173,8 +173,10 @@ func (l *Lock) toRaw() rawLock { // MarshalTOML serializes this lock into TOML via an intermediate raw form. func (l *Lock) MarshalTOML() ([]byte, error) { raw := l.toRaw() - result, err := toml.Marshal(raw) - return result, errors.Wrap(err, "Unable to marshal lock to TOML string") + var buf bytes.Buffer + enc := toml.NewEncoder(&buf).ArraysWithOneElementPerLine(true) + err := enc.Encode(raw) + return buf.Bytes(), errors.Wrap(err, "Unable to marshal lock to TOML string") } // LockFromSolution converts a gps.Solution to dep's representation of a lock. diff --git a/manifest.go b/manifest.go index 2af9c9ae4b..8dd70d7555 100644 --- a/manifest.go +++ b/manifest.go @@ -454,8 +454,10 @@ func toProject(raw rawProject) (n gps.ProjectRoot, pp gps.ProjectProperties, err // MarshalTOML serializes this manifest into TOML via an intermediate raw form. func (m *Manifest) MarshalTOML() ([]byte, error) { raw := m.toRaw() - result, err := toml.Marshal(raw) - return result, errors.Wrap(err, "unable to marshal the lock to a TOML string") + var buf bytes.Buffer + enc := toml.NewEncoder(&buf).ArraysWithOneElementPerLine(true) + err := enc.Encode(raw) + return buf.Bytes(), errors.Wrap(err, "unable to marshal the lock to a TOML string") } // toRaw converts the manifest into a representation suitable to write to the manifest file From 566dba0b73ddc5333039c0b88786912885de15b9 Mon Sep 17 00:00:00 2001 From: sam boyer Date: Mon, 18 Dec 2017 09:25:48 -0500 Subject: [PATCH 3/3] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4bd428efc..f927a3f048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ NEW FEATURES: * Allow override of cache directory location using environment variable `DEPCACHEDIR`. ([#1234](https://github.com/golang/dep/pull/1234)) * Add support for template output in `dep status`. (#1389) +* Each element in a multi-item TOML array is output on its own line. (#1461) BUG FIXES: