Azure Download Fix

.ruby-version

@@ -0,0 +1 @@
+2.7.0

.travis.yml

@@ -5,5 +5,4 @@ addons:
     packages:
       - ca-certificates
 cache: bundler
-dist: xenial
 script: xvfb-run bundle exec rake

Gemfile

@@ -6,7 +6,6 @@ gem 'bundler'
 gem 'nokogiri'
 gem 'rake'
 gem 'rubocop'
-gem 'selenium-webdriver'
 gem 'webdrivers'
 gem 'xml-simple'
 

Gemfile.lock

@@ -1,41 +1,47 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
-    childprocess (1.0.1)
-      rake (< 13.0)
-    coderay (1.1.2)
-    jaro_winkler (1.5.3)
-    method_source (0.9.2)
-    mini_portile2 (2.4.0)
-    nokogiri (1.10.8)
-      mini_portile2 (~> 2.4.0)
-    parallel (1.17.0)
-    parser (2.6.3.0)
-      ast (~> 2.4.0)
-    pry (0.12.2)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    ast (2.4.2)
+    childprocess (3.0.0)
+    coderay (1.1.3)
+    method_source (1.0.0)
+    mini_portile2 (2.5.0)
+    nokogiri (1.11.2)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
+      ast (~> 2.4.1)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    racc (1.5.2)
     rainbow (3.0.0)
-    rake (12.3.3)
-    rubocop (0.74.0)
-      jaro_winkler (~> 1.5.1)
+    rake (13.0.3)
+    regexp_parser (2.1.1)
+    rexml (3.2.4)
+    rubocop (1.12.1)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    ruby-progressbar (1.10.1)
-    rubyzip (1.3.0)
-    selenium-webdriver (3.142.3)
-      childprocess (>= 0.5, < 2.0)
-      rubyzip (~> 1.2, >= 1.2.2)
-    unicode-display_width (1.6.0)
-    webdrivers (4.1.2)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
+    rubyzip (2.3.0)
+    selenium-webdriver (3.142.7)
+      childprocess (>= 0.5, < 4.0)
+      rubyzip (>= 1.2.2)
+    unicode-display_width (2.0.0)
+    webdrivers (4.6.0)
       nokogiri (~> 1.6)
-      rubyzip (~> 1.0)
+      rubyzip (>= 1.3.0)
       selenium-webdriver (>= 3.0, < 4.0)
-    xml-simple (1.1.5)
+    xml-simple (1.1.8)
 
 PLATFORMS
   ruby
@@ -46,9 +52,8 @@ DEPENDENCIES
   pry
   rake
   rubocop
-  selenium-webdriver
   webdrivers
   xml-simple
 
 BUNDLED WITH
-   2.0.1
+   2.1.4

README.md

@@ -1,5 +1,5 @@
 # whitelist
-[![Build Status](https://travis-ci.org/stephengroat/whitelist.svg?branch=master)](https://travis-ci.org/stephengroat/whitelist)
+[![Build Status](https://travis-ci.com/stephengroat/whitelist.svg?branch=master)](https://travis-ci.com/stephengroat/whitelist)
 
 Whitelist IP Addresses & Domains
 

Rakefile

@@ -18,16 +18,16 @@ namespace :collect do
   end
 end
 
-namespace :collect do
-  task :oraclecloud do
-    page = Nokogiri::HTML(open('https://docs.cloud.oracle.com/iaas/Content/Network/Concepts/overview.htm#oci-public-ips'))
-    puts page.xpath('//table')
-    puts page.xpath('//table').each do |node|
-      puts node.txt
-    end
-    puts page.xpath('//table').length
-  end
-end
+#namespace :collect do
+#  task :oraclecloud do
+#    page = Nokogiri::HTML(open('https://docs.cloud.oracle.com/iaas/Content/Network/Concepts/overview.htm#oci-public-ips'))
+#    puts page.xpath('//table')
+#    puts page.xpath('//table').each do |node|
+#      puts node.txt
+#    end
+#    puts page.xpath('//table').length
+#  end
+#end
 
 # namespace :collect do
 #  task :ibmbluemix do

collect/azure.rb

@@ -1,37 +1,41 @@
 # frozen_string_literal: true
 
 require 'webdrivers'
-require 'xmlsimple'
+require 'json'
+
+MS_DOWNLOAD_URL = 'https://www.microsoft.com/en-us/download/confirmation.aspx?id='
 
 namespace :collect do
   task :azure, %i[region] do |_t, args|
-    list = nil
-    args.with_defaults(region: '.*')
+    [57_062, 57_064, 57_063, 56_519].each do |id|
+      args.with_defaults(region: '.*')
 
-    Dir.mktmpdir do |dir|
-      options = Selenium::WebDriver::Chrome::Options.new(args: ['no-sandbox'])
+      Dir.mktmpdir do |dir|
+        options = Selenium::WebDriver::Chrome::Options.new
 
-      prefs = {
-        prompt_for_download: false,
-        default_directory: dir.to_s
-      }
-      options.add_preference(:download, prefs)
+        options.add_preference(:download,
+                               prompt_for_download: false,
+                               directory_upgrade: true,
+                               default_directory: dir.to_s)
 
-      driver = Selenium::WebDriver.for(:chrome, options: options)
+        driver = Selenium::WebDriver.for(:chrome, options: options)
 
-      driver.get('https://www.microsoft.com/en-us/download/confirmation.aspx?id=41653')
-      list = File.open(Dir.glob("#{dir}/*.xml")[0])
-      driver.quit
-    end
+        driver.get(MS_DOWNLOAD_URL + id.to_s)
+
+        loop do
+          sleep(10) if Dir.glob("#{dir}/*.json.part").any?
+          break if Dir.glob("#{dir}/*.json").any?
+          sleep(5)
+        end
 
-    # Convert XML to Hash
-    data_hash = XmlSimple.xml_in(list)
-    data_hash['Region'].each do |prefix|
-      next unless prefix['Name'] =~ /#{args[:region]}/
-      next unless prefix.key?('IpRange')
+        list = File.open(Dir.glob("#{dir}/*.json")[0])
 
-      prefix['IpRange'].each do |subnet|
-        puts subnet['Subnet']
+        JSON.parse(list.read)['values'].each do |value|
+          value['properties']['addressPrefixes'].each do |address_prefix|
+            puts address_prefix
+          end
+        end
+        driver.quit
       end
     end
   end

collect/cachefly.rb

@@ -4,7 +4,7 @@
 
 namespace :collect do
   task :cachefly do
-    list = open('https://admin.cachefly.com/ips/rproxy.txt')
+    list = open('https://cachefly.cachefly.net/ips/rproxy.txt')
     list.each do |ip|
       puts ip
     end

collect/maxcdn.rb

@@ -4,7 +4,7 @@
 
 namespace :collect do
   task :maxcdn do
-    list = open('https://www.maxcdn.com/one/assets/ips.txt')
+    list = open('https://support.maxcdn.com/hc/en-us/article_attachments/360051920551/maxcdn_ips.txt')
     list.each do |ip|
       puts ip
     end

collect/msft.rb

@@ -5,31 +5,35 @@
 
 namespace :collect do
   task :msft do |_t, _args|
-    list = nil
-
     Dir.mktmpdir do |dir|
-      options = Selenium::WebDriver::Chrome::Options.new(args: ['no-sandbox'])
+      options = Selenium::WebDriver::Chrome::Options.new
 
-      prefs = {
-        prompt_for_download: false,
-        default_directory: dir.to_s
-      }
-      options.add_preference(:download, prefs)
+      options.add_preference(:download,
+                             prompt_for_download: false,
+                             directory_upgrade: true,
+                             default_directory: dir.to_s)
 
       driver = Selenium::WebDriver.for(:chrome, options: options)
 
       driver.get('https://www.microsoft.com/en-us/download/confirmation.aspx?id=53602')
+
+      loop do
+        sleep(10) if Dir.glob("#{dir}/*.csv.part").any?
+        break if Dir.glob("#{dir}/*.csv").any?
+        sleep(10)
+      end
+
       list = File.open(Dir.glob("#{dir}/*.csv")[0])
-      driver.quit
-    end
 
-    # Get array of arrays from CSV
-    data_hash = CSV.parse(list)
-    # Use first title row as keys for Hashmap
-    keys = data_hash.shift
-    data_hash.map! { |a| Hash[keys.zip(a)] }
-    data_hash.each do |prefix|
-      puts prefix['Prefix']
+      # Get array of arrays from CSV
+      data_hash = CSV.parse(list)
+      # Use first title row as keys for Hashmap
+      keys = data_hash.shift
+      data_hash.map! { |a| Hash[keys.zip(a)] }
+      data_hash.each do |prefix|
+        puts prefix['Prefix']
+      end
+      driver.quit
     end
   end
 end