Fix tokenization of consecutive \r\n line endings (#460)

Without this patch `\r\n\r\n` would be tokenized as `\r\n\r` and `\n`
instead of `\r\n` and `\r\n`. Fixes
fredrikekre/Runic.jl#15.

Author: fredrikekre

src/tokenize.jl

@@ -733,9 +733,10 @@ function lex_whitespace(l::Lexer, c)
         if c == '\n'
             k = K"NewlineWs"
         end
-        pc = peekchar(l)
+        pc, ppc = dpeekchar(l)
         # stop on non whitespace and limit to a single newline in a token
-        if !iswhitespace(pc) || (k == K"NewlineWs" && pc == '\n')
+        if !iswhitespace(pc) ||
+                (k == K"NewlineWs" && (pc == '\n' || (pc == '\r' && ppc == '\n')))
             break
         end
         c = readchar(l)

test/tokenize.jl

@@ -163,6 +163,16 @@ end
     @test untokenize(tok(str), str)==">>"
 end
 
+@testset "tokenize newlines" begin
+    n = "\n"
+    rn = "\r\n"
+    nl = K"NewlineWs"
+    for i in 0:5
+        j = 5 - i
+        @test toks(n^i * rn^j) == vcat(fill(n  => nl, i), fill(rn => nl, j))
+        @test toks(rn^i * n^j) == vcat(fill(rn => nl, i), fill(n  => nl, j))
+    end
+end
 
 @testset "test added operators" begin
     @test tok("1+=2",  2).kind == K"+="