From 581e662b713d32e54c54dbd58e3948db8c66f18d Mon Sep 17 00:00:00 2001 From: till Date: Mon, 13 Feb 2023 17:31:18 +0100 Subject: [PATCH] Chore: refactor - splits code into command classes - executes `do_it` as a default task - refactor status code handling - updates bundler/platform/nokogiri - handle encoding everywhere - adds test coverage (a few "online" tests) --- .github/workflows/pr.yml | 1 + Gemfile.lock | 19 ++-- Makefile | 4 +- README.md | 2 +- lib/apt/spy2.rb | 151 ++------------------------------ lib/apt/spy2/command/check.rb | 29 ++++++ lib/apt/spy2/command/command.rb | 81 +++++++++++++++++ lib/apt/spy2/command/fix.rb | 59 +++++++++++++ lib/apt/spy2/command/list.rb | 31 +++++++ lib/apt/spy2/country.rb | 2 +- lib/apt/spy2/launchpad.rb | 9 +- lib/apt/spy2/status.rb | 24 +++++ lib/apt/spy2/writer.rb | 14 +-- tests/country_test.rb | 6 +- tests/launchpad_test.rb | 14 +-- tests/status_test.rb | 21 +++++ 16 files changed, 289 insertions(+), 178 deletions(-) create mode 100644 lib/apt/spy2/command/check.rb create mode 100644 lib/apt/spy2/command/command.rb create mode 100644 lib/apt/spy2/command/fix.rb create mode 100644 lib/apt/spy2/command/list.rb create mode 100644 lib/apt/spy2/status.rb create mode 100644 tests/status_test.rb diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7c6ef64..735c691 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -45,5 +45,6 @@ jobs: - run: gem install pkg/apt-spy2* - run: apt-spy2 check --strict - run: sudo env "PATH=$PATH" apt-spy2 fix --commit --strict + - run: sudo env "PATH=$PATH" apt-spy2 check --launchpad --country=us --strict diff --git a/Gemfile.lock b/Gemfile.lock index 084be90..2fed011 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -14,10 +14,10 @@ GEM colored (1.2) docile (1.4.0) json (2.6.3) - mini_portile2 (2.8.1) minitest (5.15.0) - nokogiri (1.14.1) - mini_portile2 (~> 2.8.0) + nokogiri (1.14.2-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.14.2-x86_64-linux) racc (~> 1.4) parallel (1.22.1) parser (3.2.1.0) @@ -37,21 +37,22 @@ GEM rubocop-ast (>= 1.24.1, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.24.1) - parser (>= 3.1.1.0) + rubocop-ast (1.26.0) + parser (>= 3.2.1.0) ruby-progressbar (1.11.0) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov-lcov (0.8.0) - simplecov_json_formatter (0.1.3) + simplecov_json_formatter (0.1.4) thor (1.2.1) unicode-display_width (2.4.2) PLATFORMS - ruby + x86_64-darwin-22 + x86_64-linux DEPENDENCIES apt-spy2! @@ -63,4 +64,4 @@ DEPENDENCIES simplecov-lcov (~> 0.8.0) BUNDLED WITH - 2.2.32 + 2.3.15 diff --git a/Makefile b/Makefile index 6148f9a..f1e6a70 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,11 @@ container:=apt-spy2 clean: docker rm -f $(container) || true + rm -rf vendor install: - bundle install --path ./vendor/bundle + bundle config set --local path './vendor/bundle' + bundle install release: bundle exec rake release diff --git a/README.md b/README.md index 697a708..a6bf4ca 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ gem install apt-spy2 ## Usage ```sh -$ apt-spy2 [21:03:52] +$ apt-spy2 apt-spy2 commands: apt-spy2 check # Evaluate mirrors apt-spy2 fix # Set the closest/fastest mirror diff --git a/lib/apt/spy2.rb b/lib/apt/spy2.rb index 3608866..c49165b 100755 --- a/lib/apt/spy2.rb +++ b/lib/apt/spy2.rb @@ -1,162 +1,29 @@ # frozen_string_literal: true require 'thor' -require 'colored' -require 'fileutils' -require 'apt/spy2/writer' -require 'apt/spy2/country' -require 'apt/spy2/downloader' -require 'apt/spy2/ubuntu_mirrors' -require 'apt/spy2/launchpad' -require 'apt/spy2/request' -require 'apt/spy2/url' +require 'apt/spy2/command/fix' +require 'apt/spy2/command/list' +require 'apt/spy2/command/check' +require 'apt/spy2/version' # apt-spy2 command interface class AptSpy2 < Thor package_name 'apt-spy2' class_option :country, default: 'mirrors' - class_option :launchpad, type: :boolean, banner: "Use launchpad's mirror list" - - desc 'fix', 'Set the closest/fastest mirror' - option :commit, type: :boolean - option :strict, type: :boolean - def fix - mirrors = retrieve(options[:country], use_launchpad?(options)) - working = filter(mirrors, options[:strict], false) - print 'The closest mirror is: ' - puts (working[0]).to_s.bold.magenta - unless options[:commit] - puts 'Run with --commit to adjust /etc/apt/sources.list'.yellow - return - end - - puts 'Updating /etc/apt/sources.list'.yellow - update(working[0]) - end + class_option :launchpad, type: :boolean, banner: 'Use launchpad\'s mirror list' desc 'check', 'Evaluate mirrors' - option :output, type: :boolean, default: true - option :format, default: 'shell' - option :strict, type: :boolean - def check - @writer = Apt::Spy2::Writer.new(options[:format]) + subcommand 'check', Apt::Spy2::Command::Check - mirrors = retrieve(options[:country], use_launchpad?(options)) - filter(mirrors, options[:strict], options[:output]) - - puts @writer.to_json if @writer.json? - end + desc 'fix', 'Update sources' + subcommand 'fix', Apt::Spy2::Command::Fix desc 'list', 'List the currently available mirrors' - option :format, default: 'shell' - def list - mirrors = retrieve(options[:country], use_launchpad?(options)) - - @writer = Apt::Spy2::Writer.new(options[:format]) - - @writer.complete(mirrors) - - puts @writer.to_json if @writer.json? - puts mirrors unless @writer.json? - end + subcommand 'list', Apt::Spy2::Command::List desc 'version', 'Show which version of apt-spy2 is installed' def version puts Apt::Spy2::VERSION exit end - - private - - def retrieve(country = 'mirrors', launchpad = false) - downloader = Apt::Spy2::Downloader.new - - if launchpad - csv_path = File.expand_path("#{File.dirname(__FILE__)}/../../var/country-names.txt") - country = Apt::Spy2::Country.new(csv_path) - name = country.to_country_name(options[:country]) - - launchpad = Apt::Spy2::Launchpad.new(downloader.do_download('https://launchpad.net/ubuntu/+archivemirrors')) - return launchpad.mirrors(name) - end - - country.upcase! if country.length == 2 - - ubuntu_mirrors = Apt::Spy2::UbuntuMirrors.new(downloader.do_download("http://mirrors.ubuntu.com/#{country}.txt")) - ubuntu_mirrors.mirrors(country) - end - - def filter(mirrors, strict = false, output = true) - # f me :) - - working_mirrors = [] - - url = Apt::Spy2::Url.new(strict) - - mirrors.each do |mirror| - data = { 'mirror' => mirror } - - check = url.adjust!(mirror) - - status = broken?(check) - - data['status'] = status - - working_mirrors << mirror if status == 'up' - - @writer.echo(data) if output - end - - working_mirrors - end - - def broken?(url) - begin - req = Apt::Spy2::Request.new(url) - response = req.head - return 'up' if response.code == '200' - - return 'broken' if response.code == '404' - rescue StandardError - # connection errors, ssl errors, etc. - end - - 'down' - end - - def update(mirror) - t = Time.now - r = `lsb_release -c`.split(' ')[1] - sources = "## Updated on #{t} by apt-spy2\n" - sources += "deb #{mirror} #{r} main restricted universe multiverse\n" - sources += "deb #{mirror} #{r}-updates main restricted universe multiverse\n" - sources += "deb #{mirror} #{r}-backports main restricted universe multiverse\n" - sources += "deb #{mirror} #{r}-security main restricted universe multiverse\n" - - apt_sources = '/etc/apt/sources.list' - - begin - File.rename apt_sources, "#{apt_sources}.#{t.to_i}" - File.open(apt_sources, 'w') do |f| - f.write(sources) - end - rescue StandardError - msg = "Failed updating #{apt_sources}!" - msg += 'You probably need sudo!' - raise msg - end - - puts "Updated '#{apt_sources}' with #{mirror}".green - puts 'Run `apt-get update` to update'.black_on_yellow - end - - def use_launchpad?(options) - return false unless options[:launchpad] - - if options[:country] && options[:country] == 'mirrors' - raise 'Please supply a `--country=foo`. Launchpad cannot guess!' - end - - true - end end diff --git a/lib/apt/spy2/command/check.rb b/lib/apt/spy2/command/check.rb new file mode 100644 index 0000000..412dbaa --- /dev/null +++ b/lib/apt/spy2/command/check.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'apt/spy2/command/command' +require 'apt/spy2/writer' + +module Apt + module Spy2 + module Command + # runs `apt-spy2 check` + class Check < BaseCommand + option :output, type: :boolean, default: true + option :format, default: 'shell' + option :strict, type: :boolean + + desc 'do_it', '' + def do_it + @writer = Apt::Spy2::Writer.new(options[:format]) + + mirrors = retrieve(launchpad: use_launchpad?(options)) + filter(mirrors, strict: options[:strict], output: options[:output]) + + puts @writer.to_json if @writer.json? + end + + default_task :do_it + end + end + end +end diff --git a/lib/apt/spy2/command/command.rb b/lib/apt/spy2/command/command.rb new file mode 100644 index 0000000..20bb407 --- /dev/null +++ b/lib/apt/spy2/command/command.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'thor' +require 'colored' +require 'apt/spy2/country' +require 'apt/spy2/downloader' +require 'apt/spy2/launchpad' +require 'apt/spy2/request' +require 'apt/spy2/status' +require 'apt/spy2/ubuntu_mirrors' +require 'apt/spy2/url' +require 'apt/spy2/writer' + +module Apt + module Spy2 + module Command + # BaseCommmand for all others + class BaseCommand < Thor + # rubocop:disable Metrics/BlockLength + no_commands do + def use_launchpad?(options) + return false unless options[:launchpad] + + if options[:country] && options[:country] == 'mirrors' + raise 'Please supply a `--country=foo`. Launchpad cannot guess!' + end + + true + end + + def country_names + File.expand_path("#{File.dirname(__FILE__)}/../../../../var/country-names.txt") + end + + def retrieve(launchpad: false) + downloader = Apt::Spy2::Downloader.new + + if launchpad + name = Apt::Spy2::Country.new(country_names).to_country_name(options[:country]) + launchpad = Apt::Spy2::Launchpad.new(downloader.do_download('https://launchpad.net/ubuntu/+archivemirrors')) + return launchpad.mirrors(name) + end + + country = options[:country] + country.upcase! if country.length == 2 + + ubuntu_mirrors = Apt::Spy2::UbuntuMirrors.new(downloader.do_download("http://mirrors.ubuntu.com/#{country}.txt")) + ubuntu_mirrors.mirrors(country) + end + + def filter(mirrors, strict: false, output: true) + # f me :) + working_mirrors = [] + url = Apt::Spy2::Url.new(strict) + + mirrors.each do |mirror| + data = { 'mirror' => mirror } + data['status'] = broken?(url.adjust!(mirror)) + working_mirrors << mirror if data['status'] == Apt::Spy2::Status::UP + @writer.echo(data) if output + end + + working_mirrors + end + + def broken?(url) + begin + req = Apt::Spy2::Request.new(url) + return Apt::Spy2::Status.status?(req.head.code) + rescue StandardError + # connection errors, ssl errors, etc. + end + + Apt::Spy2::Status::DOWN + end + end + # rubocop:enable Metrics/BlockLength + end + end + end +end diff --git a/lib/apt/spy2/command/fix.rb b/lib/apt/spy2/command/fix.rb new file mode 100644 index 0000000..1a91bbc --- /dev/null +++ b/lib/apt/spy2/command/fix.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'apt/spy2/command/command' +require 'colored' +require 'fileutils' + +module Apt + module Spy2 + module Command + # runs `apt-spy2 fix` + class Fix < BaseCommand + option :commit, type: :boolean + option :strict, type: :boolean + + desc 'do_it', '' + def do_it + working = filter(retrieve(launchpad: use_launchpad?(options)), strict: options[:strict], output: false) + print 'The closest mirror is: ' + puts (working[0]).to_s.bold.magenta + unless options[:commit] + puts 'Run with --commit to adjust /etc/apt/sources.list'.yellow + return + end + + puts 'Updating /etc/apt/sources.list'.yellow + update(working[0]) + end + + default_task :do_it + + private + + def update(mirror) + t = Time.now + r = `lsb_release -c`.split(' ')[1] + sources = "## Updated on #{t} by apt-spy2\n" + sources += "deb #{mirror} #{r} main restricted universe multiverse\n" + sources += "deb #{mirror} #{r}-updates main restricted universe multiverse\n" + sources += "deb #{mirror} #{r}-backports main restricted universe multiverse\n" + sources += "deb #{mirror} #{r}-security main restricted universe multiverse\n" + + apt_sources = '/etc/apt/sources.list' + + begin + File.rename apt_sources, "#{apt_sources}.#{t.to_i}" + File.open(apt_sources, 'w') do |f| + f.write(sources) + end + rescue StandardError + raise "Failed updating #{apt_sources}! You probably need sudo!" + end + + puts "Updated '#{apt_sources}' with #{mirror}".green + puts 'Run `apt update` to update'.black_on_yellow + end + end + end + end +end diff --git a/lib/apt/spy2/command/list.rb b/lib/apt/spy2/command/list.rb new file mode 100644 index 0000000..7fdcda8 --- /dev/null +++ b/lib/apt/spy2/command/list.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'apt/spy2/command/command' +require 'apt/spy2/writer' + +module Apt + module Spy2 + module Command + # runs `apt-spy2 list` + class List < BaseCommand + option :format, default: 'shell' + desc 'do_it', 'run' + def do_it + mirrors = retrieve(launchpad: use_launchpad?(options)) + + @writer = Apt::Spy2::Writer.new(options[:format]) + @writer.complete(mirrors) + + if @writer.json? + puts @writer.to_json + return + end + + puts mirrors + end + + default_task :do_it + end + end + end +end diff --git a/lib/apt/spy2/country.rb b/lib/apt/spy2/country.rb index f693f80..5d90d2f 100644 --- a/lib/apt/spy2/country.rb +++ b/lib/apt/spy2/country.rb @@ -12,7 +12,7 @@ def to_country_name(code) code = code.upcase return capitalize!(code) unless code.length == 2 - File.open(@database).each do |line| + File.open(@database, 'r:UTF-8').each do |line| country, tld = line.split(';', 2) tld.gsub!(/\n/, '') diff --git a/lib/apt/spy2/launchpad.rb b/lib/apt/spy2/launchpad.rb index bd59657..6435ba6 100644 --- a/lib/apt/spy2/launchpad.rb +++ b/lib/apt/spy2/launchpad.rb @@ -7,14 +7,17 @@ module Spy2 # parse launchpad output class Launchpad def initialize(download) - @launchpad = download + @document = Nokogiri::HTML(download) do |c| + # rubocop:disable Layout/LineLength + c.options = Nokogiri::XML::ParseOptions::HUGE | Nokogiri::XML::ParseOptions::NONET | Nokogiri::XML::ParseOptions::RECOVER + # rubocop:enable Layout/LineLength + end end def mirrors(country) mirrors = [] - document = Nokogiri::HTML(@launchpad) - table_rows = document.xpath("//tr/th[text()='#{country}']/../following-sibling::*") + table_rows = @document.xpath("//tr/th[text()='#{country}']/../following-sibling::*") raise "Couldn't find a mirror for #{country}." if table_rows.empty? table_rows.each do |node| diff --git a/lib/apt/spy2/status.rb b/lib/apt/spy2/status.rb new file mode 100644 index 0000000..537fea3 --- /dev/null +++ b/lib/apt/spy2/status.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Apt + module Spy2 + # wraps up, down and broken + class Status + UP = 'UP' + DOWN = 'DOWN' + BROKEN = 'BROKEN' + + def self.print(status) + table = { UP => 'green', DOWN => 'red', BROKEN => 'yellow' } + puts status.send(table[status]) + end + + def self.status?(code) + return UP if code == '200' + return BROKEN if code == '404' + + DOWN + end + end + end +end diff --git a/lib/apt/spy2/writer.rb b/lib/apt/spy2/writer.rb index 18dd696..87db79c 100644 --- a/lib/apt/spy2/writer.rb +++ b/lib/apt/spy2/writer.rb @@ -2,6 +2,7 @@ require 'colored' require 'json' +require 'apt/spy2/status' module Apt module Spy2 @@ -25,17 +26,7 @@ def echo(data) end print "Mirror: #{data['mirror']} - " - - case data['status'] - when 'up' - puts data['status'].upcase.green - when 'down' - puts data['status'].upcase.red - when 'broken' - puts data['status'].upcase.yellow - else - puts "Unknown status: #{data['status']}".white_on_red - end + Apt::Spy2::Status.print(data['status']) end def json? @@ -44,6 +35,7 @@ def json? false end + # generates a json string def to_json(*_args) JSON.generate(@complete) end diff --git a/tests/country_test.rb b/tests/country_test.rb index 72eef31..b70cb15 100644 --- a/tests/country_test.rb +++ b/tests/country_test.rb @@ -5,10 +5,6 @@ # test the name resolution class CountryTest < Minitest::Test - def setup - @country_list = File.expand_path("#{File.dirname(__FILE__)}/../var/country-names.txt") - end - def test_tld_to_name # fixtures for people who don't want to read about rails data = { @@ -20,7 +16,7 @@ def test_tld_to_name 'germany' => 'Germany' } - c = Apt::Spy2::Country.new(@country_list) + c = Apt::Spy2::Country.new(File.open("#{File.dirname(File.dirname(__FILE__))}/var/country-names.txt", 'r:UTF-8')) data.each_pair do |code, expected| assert_equal(expected, c.to_country_name(code)) diff --git a/tests/launchpad_test.rb b/tests/launchpad_test.rb index d76c18a..fd95e63 100644 --- a/tests/launchpad_test.rb +++ b/tests/launchpad_test.rb @@ -2,16 +2,20 @@ require_relative '../test_helper' require_relative '../lib/apt/spy2/launchpad' +require_relative '../lib/apt/spy2/downloader' # test to confirm launchpad format (HTML) is parsed class LaunchpadTest < Minitest::Test - def setup - @download_fixture = File.read(File.expand_path("#{File.dirname(__FILE__)}/fixtures/launchpad.html")) - end - def test_german_mirrors - lp = Apt::Spy2::Launchpad.new(@download_fixture) + lp = Apt::Spy2::Launchpad.new(File.open("#{File.dirname(__FILE__)}/fixtures/launchpad.html", 'r:UTF-8')) mirrors = lp.mirrors('Germany') assert_equal(false, mirrors.empty?) end + + def test_online + downloader = Apt::Spy2::Downloader.new + launchpad = Apt::Spy2::Launchpad.new(downloader.do_download('https://launchpad.net/ubuntu/+archivemirrors')) + mirrors = launchpad.mirrors('Ukraine') + assert_equal(false, mirrors.empty?) + end end diff --git a/tests/status_test.rb b/tests/status_test.rb new file mode 100644 index 0000000..a820f88 --- /dev/null +++ b/tests/status_test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require_relative '../test_helper' +require_relative '../lib/apt/spy2/status' + +# test the name resolution +class StatusTest < Minitest::Test + def test_response_code_to_status + data = { + '200' => Apt::Spy2::Status::UP, + '404' => Apt::Spy2::Status::BROKEN, + '500' => Apt::Spy2::Status::DOWN, + '' => Apt::Spy2::Status::DOWN + } + + data.each_pair do |code, expected| + status = Apt::Spy2::Status.status?(code) + assert_equal(expected, status) + end + end +end