Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Switch to using vcs service #263

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Switch to using vcs service #263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2a3b1ee
Switch to using the vcs service
AndriiMysko Dec 11, 2020
7b56800
Debug
AndriiMysko Dec 23, 2020
aa9ed7b
Debug
AndriiMysko Dec 23, 2020
9299cec
Revert debug
AndriiMysko Dec 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Gemfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ gem 'keen'

gem 'jemalloc', git: 'https://github.com/joshk/jemalloc-rb'

gem 'gh'
gem 'aws-sdk'
gem 'actionmailer'
gem 'roadie'
Expand Down
1 change: 0 additions & 1 deletion Gemfile.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1197,7 +1197,6 @@ DEPENDENCIES
faraday (~> 1.0)
faraday_middleware
foreman
gh
guard
guard-rspec
intercom (~> 3.8.0)
Expand Down
368 changes: 62 additions & 306 deletions lib/travis/addons/github_status/task.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
require 'gh'
module Travis
module Addons
module GithubStatus
Expand Down Expand Up @@ -33,312 +32,69 @@ class Task < Travis::Task
REDIS_PREFIX = 'travis-tasks:github-status:'.freeze

private
def url
client.create_status_url(repository[:vcs_id], sha)
end

def process(timeout)
return process_vcs if repository[:vcs_type] != 'GithubRepository'.freeze
users_tried = []
status = :not_ok

message = %W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
state=#{state}
commit=#{sha}
tokens_count=#{tokens.size}
installation_id=#{installation_id}
].join(' ')

if !installation_id.nil? && process_via_github_app
info("#{message} processed_with=#{client.name}")
return
end

while !tokens.empty? and status != :ok do
username, token = tokens.shift
unless token
error("#{message} username=#{username} token=#{token.to_s}")
next
end

status, details = process_with_token(username, token)
if status == :ok
info("#{message} username=#{username} processed_with=user_token token=#{token[0,3]}...")
return
elsif status == :skipped
info "#{message} message=\"Token for #{username} failed within the last hour. Skipping\""
next
end

# we can't post any more status to this commit, so there's
# no point in trying further
return if details[:status].to_i == 422

users_tried << username
error(%W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
error=not_updated
commit=#{sha}
username=#{username}
url=#{url}
github_response=#{details[:status]}
processed_with=user_token
users_tried=#{users_tried}
last_token_tried="#{token.to_s[0,3]}..."
rate_limit=#{rate_limit_info details[:response_headers]}
github_request_id=#{github_request_id details[:response_headers]}
].join(' '))
end

error("#{message} message=\"All known tokens failed to update status\"")
end

def process_vcs
client.create_status(
process_via_gh_apps: false,
id: repository[:vcs_id],
type: repository[:vcs_type],
ref: sha,
pr_number: payload[:pull_request] && payload[:pull_request][:number],
payload: status_payload
)
end

def tokens
params.fetch(:tokens) { { '<legacy format>' => params[:token] } }
end

def process_with_token(username, token)
Travis.redis_pool.with do |redis|
if redis.exists(errored_token_key(token))
return [:skipped, {}]
end
end

value = authenticated(token) do
client.create_status(
process_via_gh_apps: false,
id: repository[:vcs_id],
type: repository[:vcs_type],
ref: sha,
payload: status_payload
)
end
[:ok, value]
rescue GH::Error(:response_status => 401),
GH::Error(:response_status => 403),
GH::Error(:response_status => 404),
GH::Error(:response_status => 422) => e
mark_token(username, token) if e.info[:response_status] == 403
error(%W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
state=#{state}
commit=#{sha}
username=#{username}
response_status=#{e.info[:response_status]}
reason=#{ERROR_REASONS.fetch(Integer(e.info[:response_status]))}
processed_with=user_token
body=#{e.info[:response_body]}
last_token_tried="#{token.to_s[0,3]}..."
rate_limit=#{rate_limit_info e.info[:response_headers]}
github_request_id=#{github_request_id e.info[:response_headers]}
].join(' '))

return [
:error,
{
status: e.info[:response_status],
reason: e.info[:response_body],
response_headers: e.info[:response_headers]
}
]
rescue GH::Error => e
message = %W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
error=not_updated
commit=#{sha}
url=#{url}
response_status=#{e.info[:response_status]}
message=#{e.message}
processed_with=user_token
body=#{e.info[:response_body]}
last_token_tried="#{token.to_s[0,3]}..."
rate_limit=#{rate_limit_info e.info[:response_headers]}
github_request_id=#{github_request_id e.info[:response_headers]}
].join(' ')
error(message)
raise message
end

def process_via_github_app
response = client.create_status(
process_via_gh_apps: true,
id: repository[:vcs_id],
type: repository[:vcs_type],
ref: sha,
payload: status_payload.to_json
)

if response.success?
info(%W[
type=github_status
repo=#{repository[:slug]}
response_status=#{response.status}
processed_with=#{client.name}
].join(' '))
return true
end

status_int = Integer(response.status)
case status_int
when 401, 403, 404, 422
error(%W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
state=#{state}
commit=#{sha}
installation_id=#{installation_id}
response_status=#{status_int}
reason=#{ERROR_REASONS.fetch(status_int)}
processed_with=#{client.name}
body=#{response.body}
].join(' '))
return nil
else
message = %W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
error=not_updated
commit=#{sha}
url=#{url}
response_status=#{status_int}
processed_with=#{client.name}
body=#{response.body}
].join(' ')
error(message)
raise message
end
end

def target_url
with_utm("#{Travis.config.http_host}/#{Travis::Addons::Util::Helpers.vcs_prefix(repository[:vcs_type])}/#{repository[:vcs_slug] || repository[:slug]}/builds/#{build[:id]}", :github_status)
end

def status_payload
{
state: state,
description: description,
target_url: target_url,
context: context
}
end

def client
@client ||= Travis::Api.backend(repository[:vcs_id], repository[:vcs_type], installation_id: installation_id)
end

def installation_id
params.fetch(:installation, nil)
end

def sha
pull_request? ? request[:head_commit] : commit[:sha]
end

def context
build_type = pull_request? ? "pr" : "push"
"continuous-integration/travis-ci/#{build_type}"
end

def state
STATES[build[:state]]
end

def description
DESCRIPTIONS[state]
end

def authenticated(token, &block)
GH.with(gh_no_tokencheck_stack.build(http_options(token)), &block)
end

def gh_no_tokencheck_stack
return @gh_no_tokencheck_stack if @gh_no_tokencheck_stack
@gh_no_tokencheck_stack = ::GH::Stack.new do
use ::GH::Instrumentation
use ::GH::Parallel
use ::GH::Pagination
use ::GH::LinkFollower
use ::GH::MergeCommit
use ::GH::LazyLoader
use ::GH::Normalizer
use ::GH::CustomLimit
use ::GH::Remote
end

@gh_no_tokencheck_stack.options.merge! ::GH::DefaultStack.options
@gh_no_tokencheck_stack
end

def http_options(token)
super().merge(token: token, headers: headers, ssl: (Travis.config.github.ssl || {}).to_hash.compact)
end

def headers
{
"Accept" => "application/vnd.github.v3+json"
}
end

def rate_limit_info(headers)
return {error: "headers were nil"} unless headers

unless rate_limit_headers_complete? headers
return {error: "response headers did not contain rate limit information"}
end

{
limit: headers["x-ratelimit-limit"].to_i,
remaining: headers["x-ratelimit-remaining"].to_i,
next_limit_reset_in: headers["x-ratelimit-reset"].to_i - Time.now.to_i
}
end

def rate_limit_headers_complete?(headers)
!headers.nil? &&
!headers["x-ratelimit-limit" ].to_s.empty? &&
!headers["x-ratelimit-remaining"].to_s.empty? &&
!headers["x-ratelimit-reset" ].to_s.empty?
end

def github_request_id(headers)
if headers.respond_to? :[]
headers["x-github-request-id"]
end
end

def errored_token_key(token)
token_hash = Digest::SHA256.hexdigest(token)
REDIS_PREFIX + "errored_tokens:#{token_hash}"
end

def mark_token(login, token)
info "message=\"A request with token belonging to #{login} failed. Will skip using this token for 1 hour.\""
Travis.redis_pool.with do |redis|
redis.set errored_token_key(token), "", ex: 60*60 # an hour
end
end
def url
client.create_status_url(repository[:vcs_id], sha)
end

def process(_timeout)
client.create_status(
id: repository[:vcs_id],
type: repository[:vcs_type],
ref: sha,
pr_number: payload[:pull_request] && payload[:pull_request][:number],
payload: status_payload
)

message = %W[
type=github_status
build=#{build[:id]}
repo=#{repository[:slug]}
state=#{state}
commit=#{sha}
installation_id=#{installation_id}
].join(' ')

info("#{message} processed_with=#{client.name}")
end

def target_url
with_utm("#{Travis.config.http_host}/#{Travis::Addons::Util::Helpers.vcs_prefix(repository[:vcs_type])}/#{repository[:vcs_slug] || repository[:slug]}/builds/#{build[:id]}", :github_status)
end

def status_payload
{
state: state,
description: description,
target_url: target_url,
context: context
}
end

def client
@client ||= Travis::Api.backend(repository[:vcs_id], repository[:vcs_type], installation_id: installation_id)
end

def installation_id
params.fetch(:installation, nil)
end

def sha
pull_request? ? request[:head_commit] : commit[:sha]
end

def context
build_type = pull_request? ? "pr" : "push"
"continuous-integration/travis-ci/#{build_type}"
end

def state
STATES[build[:state]]
end

def description
DESCRIPTIONS[state]
end
end
end
end
Expand Down
Loading