Add scripts to import geolite2 data

Author: NuckChorris

bin/util/format_geolite2_cities_csv

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require 'csv'
+require 'ipaddr'
+
+ARGV.each do |file|
+  CSV.open(file, 'r', headers: true).each do |row|
+    network = IPAddr.new(row['network']).to_range
+    next if row['geoname_id'].nil?
+    puts CSV.generate_line([
+      network.begin,
+      network.end,
+      row['geoname_id']
+    ])
+  end
+end

bin/util/format_geolite2_ipasn_csv

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require 'csv'
+require 'ipaddr'
+
+ARGV.each do |file|
+  CSV.open(file, 'r', headers: true).each do |row|
+    network = IPAddr.new(row['network']).to_range
+    puts CSV.generate_line([
+      network.begin,
+      network.end,
+      row['autonomous_system_number'],
+      row['autonomous_system_organization']
+    ])
+  end
+end

bin/util/format_geolite2_locations_csv

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require 'csv'
+
+ARGV.each do |file|
+  CSV.open(file, 'r', headers: true).each do |row|
+    puts CSV.generate_line([
+      row['geoname_id'],
+      row['continent_code'],
+      row['continent_name'],
+      row['country_iso_code'],
+      row['country_name'],
+      row['subdivision_1_iso_code'],
+      row['subdivision_1_name'],
+      row['subdivision_2_iso_code'],
+      row['subdivision_2_name'],
+      row['city_name'],
+    ])
+  end
+end

bin/util/format_import_csv_to_sql

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require 'securerandom'
+
+# This script takes a CSV file on STDIN and outputs a SQL script that will import the CSV into a new
+# table with the same schema as the specified base table, then safely hot-swap the new table in for
+# the base table before deleting it. This allows us to import a whole dataset in one go without
+# locking the table for reads.
+
+base_table = ARGV[0]
+new_table = "#{base_table}_#{SecureRandom.alphanumeric(5)}"
+old_table = "#{new_table}_old"
+
+def tee(str)
+  $stdout.puts str
+  $stderr.puts str
+end
+
+tee "BEGIN;"
+
+tee "CREATE TABLE #{new_table} (LIKE #{base_table} INCLUDING ALL);"
+tee "COPY public.#{new_table} FROM stdin WITH (FORMAT csv, FREEZE true);"
+
+$stdin.each_line { |line| puts line }
+
+puts "\\."
+
+tee "ALTER TABLE #{base_table} RENAME TO #{old_table};"
+tee "ALTER TABLE #{new_table} RENAME TO #{base_table};"
+tee "DROP TABLE #{old_table} CASCADE;"
+
+tee "COMMIT;"

bin/util/import_geolite2

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+auth="$1"
+
+function download_geolite2() {
+  set -e
+  local tmpfile
+  local tmpdir
+  tmpfile="$(mktemp -u)"
+  tmpdir="$(mktemp -d)"
+  curl -o "$tmpfile" -J -L -u "$auth" --fail-with-body\
+    "https://download.maxmind.com/geoip/databases/GeoLite2-$1-CSV/download?suffix=zip" >&2
+  unzip "$tmpfile" -d "$tmpdir" >&2
+  rm "$tmpfile"
+  echo "$tmpdir"/GeoLite2-"$1"-CSV_*/
+}
+
+citydir=$(download_geolite2 City)
+asndir=$(download_geolite2 ASN)
+
+./import_geolite2_cities "$citydir"/GeoLite2-City-Blocks-IPv{4,6}.csv
+./import_geolite2_locations "$citydir"/GeoLite2-City-Locations-en.csv
+./import_geolite2_ipasn "$asndir"/GeoLite2-ASN-Blocks-IPv{4,6}.csv

bin/util/import_geolite2_cities

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+./format_geolite2_cities_csv "$@" | ./format_import_csv_to_sql ip_cities | (
+  cd ../../ && ./bin/rails dbconsole
+)

bin/util/import_geolite2_ipasn

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+./format_geolite2_ipasn_csv "$@" | ./format_import_csv_to_sql ip_asns | (
+  cd ../../ && ./bin/rails dbconsole
+)

bin/util/import_geolite2_locations

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "$0")"
+
+./format_geolite2_locations_csv "$@" | ./format_import_csv_to_sql geonames | (
+  cd ../../ && ./bin/rails dbconsole
+)