Commit 2f8e2c03 authored by Robert Speicher's avatar Robert Speicher 🇲🇽

Replace HTTParty with Faraday; implement Direction page caching

This will cache all of the API responses required by the Direction page
generation for 24 hours, which should significantly speed up the build
process.
parent 73ac930b
......@@ -13,6 +13,7 @@ before_script: [*install]
cache:
key: "web"
paths:
- tmp/cache
- vendor
stages:
......
......@@ -18,6 +18,8 @@ gem 'middleman-syntax'
gem 'kramdown', '~> 1.10'
gem 'nokogiri'
gem 'sassc'
gem 'stringex'
# Replacement of therubyracer
gem 'mini_racer', '~> 0.1'
......@@ -25,9 +27,10 @@ gem 'mini_racer', '~> 0.1'
# For feed.xml.builder
gem 'builder', '~> 3.0'
gem 'httparty'
gem 'stringex'
gem 'sassc'
# Direction generation
gem 'faraday', '>= 0.15.0'
gem 'faraday_middleware'
gem 'faraday_middleware-parse_oj', '~> 0.3'
group :development, :test do
gem 'html-proofer'
......
......@@ -33,6 +33,14 @@ GEM
ffi (>= 1.3.0)
eventmachine (1.2.3)
execjs (2.7.0)
faraday (0.15.2)
multipart-post (>= 1.2, < 3)
faraday_middleware (0.12.2)
faraday (>= 0.7.4, < 1.0)
faraday_middleware-parse_oj (0.3.2)
faraday (~> 0.9)
faraday_middleware (>= 0.9.1, < 1.0)
oj (>= 2.0, < 4.0)
fast_blank (1.0.0)
fastimage (2.1.0)
ffi (1.9.18)
......@@ -52,8 +60,6 @@ GEM
yell (~> 2.0)
htmlcompressor (0.2.0)
http_parser.rb (0.6.0)
httparty (0.14.0)
multi_xml (>= 0.5.2)
i18n (0.7.0)
kramdown (1.13.2)
libv8 (5.9.211.38.1)
......@@ -116,9 +122,10 @@ GEM
mini_racer (0.1.14)
libv8 (~> 5.9)
minitest (5.10.2)
multi_xml (0.6.0)
multipart-post (2.0.0)
nokogiri (1.7.1)
mini_portile2 (~> 2.1.0)
oj (3.6.2)
padrino-helpers (0.13.3.3)
i18n (~> 0.6, >= 0.6.7)
padrino-support (= 0.13.3.3)
......@@ -190,8 +197,10 @@ PLATFORMS
DEPENDENCIES
builder (~> 3.0)
docopt
faraday (>= 0.15.0)
faraday_middleware
faraday_middleware-parse_oj (~> 0.3)
html-proofer
httparty
kramdown (~> 1.10)
middleman (~> 4.2)
middleman-autoprefixer
......@@ -211,4 +220,4 @@ DEPENDENCIES
yaml-lint
BUNDLED WITH
1.16.0
1.16.1
require 'httparty'
require 'active_support/cache'
require 'active_support/core_ext/date/calculations'
require 'active_support/core_ext/numeric/time'
require 'active_support/notifications'
require 'faraday_middleware'
require 'faraday_middleware/parse_oj'
PRIVATE_TOKEN = ENV['PRIVATE_TOKEN']
class GitLabInstance
attr_reader :name, :endpoint
attr_reader :name
def initialize(name)
raise 'PRIVATE_TOKEN required to generate direction page' unless ENV['PRIVATE_TOKEN']
@connection = Faraday.new(url: endpoint, headers: headers, request: { timeout: 90 }) do |config|
config.request :json
config.response :logger, nil, { headers: false, bodies: false } unless ENV['CI']
config.response :oj
config.adapter Faraday.default_adapter
end
def initialize(endpoint, private_token, name)
@endpoint = "#{endpoint}/api/v4"
@private_token = private_token
@name = name
end
def call(path, params = '')
url = @endpoint + path + params
response = HTTParty.get(url, headers: headers)
def get(path, params = {})
cache_key = @connection.build_url(path, params).to_s
if response.ok?
response
else
puts "#{response.code} while retrieving #{url}"
response = cache_store.read(cache_key)
# Unless the cached response exists and was successful, perform the request
if response.nil? || response.status != 200
begin
response = @connection.get(path, params)
[]
if response.status == 200
cache_store.write(cache_key, response)
if response.headers['X-Total-Pages']&.to_i > 1
warn "WARNING: More than one page returned by #{response.env.url}"
end
else
warn "Error in retrieving URL #{response.env.url}: #{response.status}"
end
rescue Faraday::ClientError => ex
warn "Error in retrieving URL #{response.env.url}: #{ex.inspect}"
end
end
response.body
end
private
def endpoint
'https://gitlab.com/api/v4'
end
def headers
{
'PRIVATE-TOKEN' => @private_token,
'PRIVATE-TOKEN' => PRIVATE_TOKEN,
'User-Agent' => 'www-gitlab-com'
}
end
def cache_store
@cache_store ||= ActiveSupport::Cache::FileStore.new(
File.expand_path('../tmp/cache/direction', __dir__),
expires_in: 24.hours.to_i
)
end
end
class GitLabProject
......@@ -42,7 +80,7 @@ class GitLabProject
end
def milestones
result = @instance.call("/projects/#{@id}/milestones?state=active")
result = @instance.get("projects/#{@id}/milestones", state: 'active')
result = result.select { |ms| ms['due_date'] }
result.sort_by! do |ms|
Date.parse ms['due_date']
......@@ -50,27 +88,27 @@ class GitLabProject
end
def milestone(milestone_id)
@instance.call("/projects/#{@id}/milestones/#{milestone_id}")
@instance.get("projects/#{@id}/milestones/#{milestone_id}")
end
def milestone_direction_issues(milestone_id)
@instance.call("/projects/#{@id}/issues", "?milestone=#{milestone_id}&labels=direction")
@instance.get("projects/#{@id}/issues", milestone: milestone_id, labels: 'direction')
end
def wishlist_issues(label, not_label = nil)
result = @instance.call("/projects/#{@id}/issues", "?labels=direction,#{label}&state=opened&per_page=100&sort=asc")
result = @instance.get("projects/#{@id}/issues", labels: "direction,#{label}", state: 'opened', per_page: 100, sort: 'asc')
result = result.select { |issue| (issue['labels'] & not_label).empty? } if not_label
result.select { |issue| issue['milestone'].nil? || issue['milestone']['title'] == 'Backlog' }
end
def product_vision_issues(label, not_label = nil)
result = @instance.call("/projects/#{@id}/issues", "?labels=Product+Vision+2018,#{label}&per_page=100&sort=asc")
result = @instance.get("projects/#{@id}/issues", labels: "Product+Vision+2018,#{label}", per_page: 100, sort: 'asc')
result = result.select { |issue| (issue['labels'] & not_label).empty? } if not_label
result
end
def tier_issues(tier)
@instance.call("/projects/#{@id}/issues", "?labels=direction,#{tier}&state=opened&per_page=100&sort=asc")
@instance.get("projects/#{@id}/issues", labels: "direction,#{tier}", state: 'opened', per_page: 100, sort: 'asc')
# result = result.select { |issue| issue["milestone"] && issue["milestone"]["due_date"] }
end
......@@ -83,11 +121,7 @@ class GitLabProject
end
def project
@project ||= @instance.call("/projects/#{@id}")
end
def endpoint
@instance.endpoint
@project ||= @instance.get("projects/#{@id}")
end
end
......@@ -99,7 +133,7 @@ class GitLabGroup
end
def milestones
result = @instance.call("/groups/#{@id}/milestones?state=active")
result = @instance.get("groups/#{@id}/milestones", state: "active")
result = result.select { |ms| ms['due_date'] }
result.sort_by! do |ms|
Date.parse ms['due_date']
......@@ -107,13 +141,13 @@ class GitLabGroup
end
def product_vision_epics(label, not_label = nil)
result = @instance.call("/groups/#{@id}/epics", "?labels=Product+Vision+2018,#{label}&per_page=100&sort=asc")
result = @instance.get("groups/#{@id}/epics", labels: "Product+Vision+2018,#{label}", per_page: 100, sort: 'asc')
result = result.select { |epic| (epic['labels'] & not_label).empty? } if not_label
result
end
def group
@group ||= @instance.call("/groups/#{@id}")
@group ||= @instance.get("groups/#{@id}")
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment