Merge pull request #2169 from github/api-pagination

Implement `hub api --paginate`

Author: mislav

commands/api.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -66,12 +67,20 @@ var cmdApi = &Command{
 		Parse response JSON and output the data in a line-based key-value format
 		suitable for use in shell scripts.
 
+	--paginate
+		Automatically request and output the next page of results until all
+		resources have been listed. For GET requests, this follows the '<next>'
+		resource as indicated in the "Link" response header. For GraphQL queries,
+		this utilizes 'pageInfo' that must be present in the query; see EXAMPLES.
+
+		Note that multiple JSON documents will be output as a result.
+
 	--color[=<WHEN>]
 		Enable colored output even if stdout is not a terminal. <WHEN> can be one
 		of "always" (default for '--color'), "never", or "auto" (default).
 
 	--cache <TTL>
-		Cache successful responses to GET requests for <TTL> seconds.
+		Cache valid responses to GET requests for <TTL> seconds.
 
 		When using "graphql" as <ENDPOINT>, caching will apply to responses to POST
 		requests as well. Just make sure to not use '--cache' for any GraphQL
@@ -101,6 +110,22 @@ var cmdApi = &Command{
 		# perform a GraphQL query read from a file
 		$ hub api graphql -F query=@path/to/myquery.graphql
 
+		# perform pagination with GraphQL
+		$ hub api --paginate graphql -f query=''
+		  query($endCursor: String) {
+		    repositoryOwner(login: "USER") {
+		      repositories(first: 100, after: $endCursor) {
+		        nodes {
+		          nameWithOwner
+		        }
+		        pageInfo {
+		          hasNextPage
+		          endCursor
+		        }
+		      }
+		    }
+		  }''
+
 ## See also:
 
 hub(1)
@@ -165,7 +190,8 @@ func apiCommand(cmd *Command, args *Args) {
 		host = defHost.Host
 	}
 
-	if path == "graphql" && params["query"] != nil {
+	isGraphQL := path == "graphql"
+	if isGraphQL && params["query"] != nil {
 		query := params["query"].(string)
 		query = strings.Replace(query, quote("{owner}"), quote(owner), 1)
 		query = strings.Replace(query, quote("{repo}"), quote(repo), 1)
@@ -203,36 +229,69 @@ func apiCommand(cmd *Command, args *Args) {
 	}
 
 	gh := github.NewClient(host)
-	response, err := gh.GenericAPIRequest(method, path, body, headers, cacheTTL)
-	utils.Check(err)
-
-	args.NoForward()
 
 	out := ui.Stdout
 	colorize := colorizeOutput(args.Flag.HasReceived("--color"), args.Flag.Value("--color"))
-	success := response.StatusCode < 300
 	parseJSON := args.Flag.Bool("--flat")
+	includeHeaders := args.Flag.Bool("--include")
+	paginate := args.Flag.Bool("--paginate")
 
-	if !success {
-		jsonType, _ := regexp.MatchString(`[/+]json(?:;|$)`, response.Header.Get("Content-Type"))
-		parseJSON = parseJSON && jsonType
-	}
+	args.NoForward()
 
-	if args.Flag.Bool("--include") {
-		fmt.Fprintf(out, "%s %s\r\n", response.Proto, response.Status)
-		response.Header.Write(out)
-		fmt.Fprintf(out, "\r\n")
-	}
+	requestLoop := true
+	for requestLoop {
+		response, err := gh.GenericAPIRequest(method, path, body, headers, cacheTTL)
+		utils.Check(err)
+		success := response.StatusCode < 300
 
-	if parseJSON {
-		utils.JSONPath(out, response.Body, colorize)
-	} else {
-		io.Copy(out, response.Body)
-	}
-	response.Body.Close()
+		jsonType := true
+		if !success {
+			jsonType, _ = regexp.MatchString(`[/+]json(?:;|$)`, response.Header.Get("Content-Type"))
+		}
+
+		if includeHeaders {
+			fmt.Fprintf(out, "%s %s\r\n", response.Proto, response.Status)
+			response.Header.Write(out)
+			fmt.Fprintf(out, "\r\n")
+		}
+
+		endCursor := ""
+		hasNextPage := false
+
+		if parseJSON && jsonType {
+			hasNextPage, endCursor = utils.JSONPath(out, response.Body, colorize)
+		} else if paginate && isGraphQL {
+			bodyCopy := &bytes.Buffer{}
+			io.Copy(out, io.TeeReader(response.Body, bodyCopy))
+			hasNextPage, endCursor = utils.JSONPath(ioutil.Discard, bodyCopy, false)
+		} else {
+			io.Copy(out, response.Body)
+		}
+		response.Body.Close()
+
+		if !success {
+			os.Exit(22)
+		}
 
-	if !success {
-		os.Exit(22)
+		requestLoop = false
+		if paginate {
+			if isGraphQL && hasNextPage && endCursor != "" {
+				if v, ok := params["variables"]; ok {
+					variables := v.(map[string]interface{})
+					variables["endCursor"] = endCursor
+				} else {
+					variables := map[string]interface{}{"endCursor": endCursor}
+					params["variables"] = variables
+				}
+				requestLoop = true
+			} else if nextLink := response.Link("next"); nextLink != "" {
+				path = nextLink
+				requestLoop = true
+			}
+		}
+		if requestLoop && !parseJSON {
+			fmt.Fprintf(out, "\n")
+		}
 	}
 }
 

commands/commands.go

@@ -122,6 +122,7 @@ func (c *Command) HelpText() string {
 	}
 
 	long = strings.Replace(long, "'", "`", -1)
+	long = strings.Replace(long, "``", "'", -1)
 	headingRe := regexp.MustCompile(`(?m)^(## .+):$`)
 	long = headingRe.ReplaceAllString(long, "$1")
 

features/api.feature

@@ -109,6 +109,46 @@ Feature: hub api
       {"name":"Faye"}
       """
 
+  Scenario: Paginate REST
+    Given the GitHub API server:
+      """
+      get('/comments') {
+        assert :per_page => "6"
+        page = (params[:page] || 1).to_i
+        response.headers["Link"] = %(<#{request.url}&page=#{page+1}>; rel="next") if page < 3
+        json [{:page => page}]
+      }
+      """
+    When I successfully run `hub api --paginate comments?per_page=6`
+    Then the output should contain exactly:
+      """
+      [{"page":1}]
+      [{"page":2}]
+      [{"page":3}]
+      """
+
+  Scenario: Paginate GraphQL
+    Given the GitHub API server:
+      """
+      post('/graphql') {
+        variables = params[:variables] || {}
+        page = (variables["endCursor"] || 1).to_i
+        json :data => {
+          :pageInfo => {
+            :hasNextPage => page < 3,
+            :endCursor => (page+1).to_s
+          }
+        }
+      }
+      """
+    When I successfully run `hub api --paginate graphql -f query=QUERY`
+    Then the output should contain exactly:
+      """
+      {"data":{"pageInfo":{"hasNextPage":true,"endCursor":"2"}}}
+      {"data":{"pageInfo":{"hasNextPage":true,"endCursor":"3"}}}
+      {"data":{"pageInfo":{"hasNextPage":false,"endCursor":"4"}}}
+      """
+
   Scenario: Avoid leaking token to a 3rd party
     Given the GitHub API server:
       """

github/client.go

@@ -29,11 +29,12 @@ func NewClient(h string) *Client {
 }
 
 func NewClientWithHost(host *Host) *Client {
-	return &Client{host}
+	return &Client{Host: host}
 }
 
 type Client struct {
-	Host *Host
+	Host         *Host
+	cachedClient *simpleClient
 }
 
 func (client *Client) FetchPullRequests(project *Project, filterParams map[string]interface{}, limit int, filter func(*PullRequest) bool) (pulls []PullRequest, err error) {
@@ -936,6 +937,11 @@ func (client *Client) simpleApi() (c *simpleClient, err error) {
 		return
 	}
 
+	if client.cachedClient != nil {
+		c = client.cachedClient
+		return
+	}
+
 	c = client.apiClient()
 	c.PrepareRequest = func(req *http.Request) {
 		clientDomain := normalizeHost(client.Host.Host)
@@ -947,6 +953,8 @@ func (client *Client) simpleApi() (c *simpleClient, err error) {
 			req.Header.Set("Authorization", "token "+client.Host.AccessToken)
 		}
 	}
+
+	client.cachedClient = c
 	return
 }
 

utils/json.go

@@ -29,10 +29,7 @@ func stateKey(s *state) string {
 	}
 }
 
-func printValue(token json.Token) {
-}
-
-func JSONPath(out io.Writer, src io.Reader, colorize bool) {
+func JSONPath(out io.Writer, src io.Reader, colorize bool) (hasNextPage bool, endCursor string) {
 	dec := json.NewDecoder(src)
 	dec.UseNumber()
 
@@ -84,17 +81,24 @@ func JSONPath(out io.Writer, src io.Reader, colorize bool) {
 				switch tt := token.(type) {
 				case string:
 					fmt.Fprintf(out, "%s\n", strings.Replace(tt, "\n", "\\n", -1))
+					if strings.HasSuffix(k, ".pageInfo.endCursor") {
+						endCursor = tt
+					}
 				case json.Number:
 					fmt.Fprintf(out, "%s\n", color("0;35", tt))
 				case nil:
 					fmt.Fprintf(out, "\n")
 				case bool:
 					fmt.Fprintf(out, "%s\n", color("1;33", fmt.Sprintf("%v", tt)))
+					if strings.HasSuffix(k, ".pageInfo.hasNextPage") {
+						hasNextPage = tt
+					}
 				default:
 					panic("unknown type")
 				}
 				postEmit()
 			}
 		}
 	}
+	return
 }