diff --git a/Gemfile b/Gemfile
index c8ee723..789230b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -10,6 +10,7 @@ group :test do
   gem 'smart_proxy', :git => "https://github.com/theforeman/smart-proxy", :branch => "develop"
 
   gem 'minitest'
+  gem 'minitest-reporters'
   gem 'mocha'
   gem 'public_suffix'
   gem 'rack-test'
diff --git a/Rakefile b/Rakefile
index 984d9b1..b50da95 100644
--- a/Rakefile
+++ b/Rakefile
@@ -22,6 +22,22 @@ task :test do
   Rake::Task['test:core'].invoke
 end
 
+begin
+  require 'minitest/reporters'
+rescue LoadError
+  # test group not enabled
+else
+  desc 'Set up minitest for CI'
+  task 'ci:setup:minitest' do
+    Minitest::Reporters.use!
+  end
+
+  namespace :jenkins do
+    desc nil # No description means it's not listed in rake -T
+    task unit: [:test]
+  end
+end
+
 if defined? RuboCop
   desc 'Run RuboCop on the lib directory'
   RuboCop::RakeTask.new(:rubocop) do |task|