Enable caching / remove wad bits
This commit is contained in:
David Cramer
parent
3761bf3ab3
commit
a2ce5c2c30
17
.travis.yml
17
.travis.yml
|
|
@ -7,6 +7,11 @@ addons:
|
|||
packages:
|
||||
- libevent-dev
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- node_modules
|
||||
- .pip_download_cache
|
||||
|
||||
python:
|
||||
- "2.6"
|
||||
- "2.7"
|
||||
|
|
@ -25,14 +30,6 @@ env:
|
|||
- 'DJANGO="-e git+git://github.com/django/django.git#egg=Django"'
|
||||
global:
|
||||
- 'PIP_DOWNLOAD_CACHE=".pip_download_cache"'
|
||||
- 'WAD_FILES="package.json,Makefile,setup.py"'
|
||||
- 'WAD_INSTALL_COMMAND=ci/setup'
|
||||
|
||||
# This should be specified with:
|
||||
# travis encrypt S3_BUCKET_NAME=secretvalue S3_CREDENTIALS=accesskey:secretkey
|
||||
# - S3_BUCKET_NAME=
|
||||
# - S3_CREDENTIALS=
|
||||
- secure: "QEFKt0HhaMlCW0FCLTz3NDY/P4UuVl1Pw8kSUVKKjbaKn2jZdaLKS68yl3Vyrd9AqrBfRNNLBAnvpZjYL6EwqqnZzpRH3KnAf0WRxE6d5ytrUkwZtOnQaN0Tumuqc3xnoXXalPXvs+abhXZpjfOzx0oPBa2WbtMF/RJZ/bwsTKE="
|
||||
|
||||
before_install:
|
||||
# Use closer nameservers
|
||||
|
|
@ -40,11 +37,9 @@ before_install:
|
|||
# These need to be here and not in the env hash because they need to be
|
||||
# evaluated after the virtualenv has been setup
|
||||
- mkdir -p $PIP_DOWNLOAD_CACHE
|
||||
- 'export WAD_ENVIRONMENT_VARIABLES="TRAVIS_PYTHON_VERSION,TRAVIS_NODE_VERSION,WAD_CACHE_PATH"'
|
||||
- 'export WAD_CACHE_PATH="node_modules,$PIP_DOWNLOAD_CACHE,$VIRTUAL_ENV"'
|
||||
|
||||
install:
|
||||
- time ci/wad
|
||||
- time ci/setup
|
||||
|
||||
script:
|
||||
- "if [[ ${TRAVIS_PYTHON_VERSION} != 'pypy' ]]; then make lint; fi"
|
||||
|
|
|
|||
334
ci/wad
334
ci/wad
|
|
@ -1,334 +0,0 @@
|
|||
#!/usr/bin/env ruby
|
||||
|
||||
# Generated on: 20-09-2013 at 12:38
|
||||
|
||||
require 'time'
|
||||
require 'net/http'
|
||||
require 'net/https'
|
||||
require 'digest/md5'
|
||||
require 'digest/sha1'
|
||||
require 'fileutils'
|
||||
require 'openssl'
|
||||
require 'base64'
|
||||
require 'cgi'
|
||||
class Presss
|
||||
# Computes the Authorization header for a AWS request based on a message,
|
||||
# the access key ID and secret access key.
|
||||
class Authorization
|
||||
attr_accessor :access_key_id, :secret_access_key
|
||||
|
||||
def initialize(access_key_id, secret_access_key)
|
||||
@access_key_id, @secret_access_key = access_key_id, secret_access_key
|
||||
end
|
||||
|
||||
# Returns the value for the Authorization header for a message contents.
|
||||
def header(string)
|
||||
'AWS ' + access_key_id + ':' + sign(string)
|
||||
end
|
||||
|
||||
# Returns a signature for a AWS request message.
|
||||
def sign(string)
|
||||
Base64.encode64(hmac_sha1(string)).strip
|
||||
end
|
||||
|
||||
def hmac_sha1(string)
|
||||
OpenSSL::HMAC.digest('sha1', secret_access_key, string)
|
||||
end
|
||||
end
|
||||
|
||||
class HTTP
|
||||
attr_accessor :config
|
||||
|
||||
def initialize(config)
|
||||
@config = config
|
||||
end
|
||||
|
||||
# Returns the configured bucket name.
|
||||
def bucket_name
|
||||
config[:bucket_name]
|
||||
end
|
||||
|
||||
def region
|
||||
config[:region] || 'us-east-1'
|
||||
end
|
||||
|
||||
def domain
|
||||
case region
|
||||
when 'us-east-1'
|
||||
's3.amazonaws.com'
|
||||
else
|
||||
's3-%s.amazonaws.com' % region
|
||||
end
|
||||
end
|
||||
|
||||
def bucket_in_hostname?
|
||||
config[:bucket_in_hostname]
|
||||
end
|
||||
|
||||
def url_prefix
|
||||
if bucket_in_hostname?
|
||||
"https://#{bucket_name}.#{domain}"
|
||||
else
|
||||
"https://#{domain}/#{bucket_name}"
|
||||
end
|
||||
end
|
||||
|
||||
# Returns the absolute path based on the key for the object.
|
||||
def absolute_path(path)
|
||||
path.start_with?('/') ? path : '/' + path
|
||||
end
|
||||
|
||||
# Returns the canonicalized resource used in the authorization
|
||||
# signature for an absolute path to an object.
|
||||
def canonicalized_resource(path)
|
||||
if bucket_name.nil?
|
||||
raise ArgumentError, "Please configure a bucket_name: Presss.config = { bucket_name: 'my-bucket-name }"
|
||||
else
|
||||
'/' + bucket_name + absolute_path(path)
|
||||
end
|
||||
end
|
||||
|
||||
# Returns a Presss::Authorization instance for the configured
|
||||
# AWS credentials.
|
||||
def authorization
|
||||
@authorization ||= Presss::Authorization.new(
|
||||
config[:access_key_id],
|
||||
config[:secret_access_key]
|
||||
)
|
||||
end
|
||||
|
||||
def signed_url(verb, expires, headers, path)
|
||||
path = absolute_path(path)
|
||||
canonical_path = canonicalized_resource(path)
|
||||
signature = [ verb.to_s.upcase, nil, nil, expires, [ headers, canonical_path ].flatten.compact ].flatten.join("\n")
|
||||
signed = authorization.sign(signature)
|
||||
"#{url_prefix}#{path}?Signature=#{CGI.escape(signed)}&Expires=#{expires}&AWSAccessKeyId=#{CGI.escape(authorization.access_key_id)}"
|
||||
end
|
||||
|
||||
def download(path, destination)
|
||||
url = signed_url(:get, Time.now.to_i + 600, nil, path)
|
||||
Presss.log "signed_url=#{url}"
|
||||
system 'curl', '-f', '-S', '-o', destination, url
|
||||
$?.success?
|
||||
end
|
||||
|
||||
# Puts an object with a key using a file or string. Optionally pass in
|
||||
# the content-type if you want to set a specific one.
|
||||
def put(path, file)
|
||||
header = 'x-amz-storage-class:REDUCED_REDUNDANCY'
|
||||
url = signed_url(:put, Time.now.to_i + 600, header, path)
|
||||
Presss.log "signed_url=#{url}"
|
||||
system 'curl', '-f', '-S', '-H', header, '-T', file, url
|
||||
$?.success?
|
||||
end
|
||||
end
|
||||
|
||||
class << self
|
||||
attr_accessor :config
|
||||
attr_accessor :logger
|
||||
end
|
||||
self.config = {}
|
||||
|
||||
# Get a object with a certain key.
|
||||
def self.download(path, destination)
|
||||
t0 = Time.now
|
||||
request = Presss::HTTP.new(config)
|
||||
log("Trying to GET #{path}")
|
||||
if request.download(path, destination)
|
||||
puts("[wad] Downloaded in #{(Time.now - t0).to_i} seconds")
|
||||
true
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
# Puts an object with a key using a file or string. Optionally pass in
|
||||
# the content-type if you want to set a specific one.
|
||||
def self.put(path, filename, content_type='application/x-download')
|
||||
request = Presss::HTTP.new(config)
|
||||
log("Trying to PUT #{path}")
|
||||
request.put(path, filename)
|
||||
end
|
||||
|
||||
# Logs to the configured logger if a logger was configured.
|
||||
def self.log(message)
|
||||
if logger
|
||||
logger.info('[Presss] ' + message)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Utility class to push and fetch Bundler directories to speed up
|
||||
# test runs on Travis-CI
|
||||
class Wad
|
||||
class Key
|
||||
def default_environment_variables
|
||||
[]
|
||||
end
|
||||
|
||||
def default_files
|
||||
[ "#{ENV['BUNDLE_GEMFILE']}.lock" ]
|
||||
end
|
||||
|
||||
def environment_variables
|
||||
if ENV['WAD_ENVIRONMENT_VARIABLES']
|
||||
ENV['WAD_ENVIRONMENT_VARIABLES'].split(',')
|
||||
else
|
||||
default_environment_variables
|
||||
end
|
||||
end
|
||||
|
||||
def files
|
||||
ENV['WAD_FILES'] ? ENV['WAD_FILES'].split(',') : default_files
|
||||
end
|
||||
|
||||
def environment_variable_contents
|
||||
environment_variables.map { |v| ENV[v] }
|
||||
end
|
||||
|
||||
def file_contents
|
||||
files.map { |f| File.read(f) rescue nil }
|
||||
end
|
||||
|
||||
def contents
|
||||
segments = [ RUBY_VERSION, RUBY_PLATFORM ] + environment_variable_contents + file_contents
|
||||
Digest::SHA1.hexdigest(segments.join("\n"))
|
||||
end
|
||||
end
|
||||
|
||||
def initialize
|
||||
s3_configure
|
||||
end
|
||||
|
||||
def project_root
|
||||
Dir.pwd
|
||||
end
|
||||
|
||||
def artifact_name
|
||||
@artifact_name ||= Key.new.contents
|
||||
end
|
||||
|
||||
def bzip_filename
|
||||
File.join(project_root, "tmp/#{artifact_name}.tar.bz2")
|
||||
end
|
||||
|
||||
def cache_path
|
||||
ENV['WAD_CACHE_PATH'] ? ENV['WAD_CACHE_PATH'].split(",") : [ '.bundle' ]
|
||||
end
|
||||
|
||||
def s3_bucket_name
|
||||
if bucket = ENV['WAD_S3_BUCKET_NAME'] || ENV['S3_BUCKET_NAME']
|
||||
bucket
|
||||
end
|
||||
end
|
||||
|
||||
def s3_credentials
|
||||
if creds = ENV['WAD_S3_CREDENTIALS'] || ENV['S3_CREDENTIALS']
|
||||
creds.split(':')
|
||||
end
|
||||
end
|
||||
|
||||
def s3_access_key_id
|
||||
s3_credentials && s3_credentials[0]
|
||||
end
|
||||
|
||||
def s3_secret_access_key
|
||||
s3_credentials && s3_credentials[1]
|
||||
end
|
||||
|
||||
def s3_path
|
||||
"#{artifact_name}.tar.bz2"
|
||||
end
|
||||
|
||||
def s3_configure
|
||||
Presss.config = {
|
||||
:bucket_name => s3_bucket_name,
|
||||
:access_key_id => s3_access_key_id,
|
||||
:secret_access_key => s3_secret_access_key,
|
||||
:region => ENV['WAD_AWS_REGION'],
|
||||
:bucket_in_hostname => (ENV['WAD_BUCKET_IN_HOSTNAME'] == 'true')
|
||||
}
|
||||
end
|
||||
|
||||
def s3_write
|
||||
log "Trying to write Wad to S3"
|
||||
if Presss.put(s3_path, bzip_filename)
|
||||
log "Wrote Wad to S3"
|
||||
else
|
||||
log "Failed to write to S3, debug with `wad -v'"
|
||||
end
|
||||
end
|
||||
|
||||
def s3_read
|
||||
if File.exist?(bzip_filename)
|
||||
log "Removing bundle from filesystem"
|
||||
FileUtils.rm_f(bzip_filename)
|
||||
end
|
||||
|
||||
log "Trying to fetch Wad from S3"
|
||||
FileUtils.mkdir_p(File.dirname(bzip_filename))
|
||||
Presss.download(s3_path, bzip_filename)
|
||||
end
|
||||
|
||||
def zip
|
||||
log "Creating artifact with tar (#{File.basename(bzip_filename)})"
|
||||
system("cd #{project_root} && tar -cPjf #{bzip_filename} #{cache_path.join(' ')}")
|
||||
$?.success?
|
||||
end
|
||||
|
||||
|
||||
def unzip
|
||||
log "Unpacking artifact with tar (#{File.basename(bzip_filename)})"
|
||||
system("cd #{project_root} && tar -xPjf #{bzip_filename}")
|
||||
$?.success?
|
||||
end
|
||||
|
||||
def put
|
||||
zip
|
||||
s3_write
|
||||
end
|
||||
|
||||
def get
|
||||
if s3_read
|
||||
unzip
|
||||
end
|
||||
end
|
||||
|
||||
def default_command
|
||||
bundle_without = ENV['WAD_BUNDLE_WITHOUT'] || "development production"
|
||||
"bundle install --path .bundle --without='#{bundle_without}'"
|
||||
end
|
||||
|
||||
def install
|
||||
log "Installing..."
|
||||
command = ENV['WAD_INSTALL_COMMAND'] || default_command
|
||||
puts command
|
||||
system(command)
|
||||
$?.success?
|
||||
end
|
||||
|
||||
def setup
|
||||
if !s3_credentials || !s3_bucket_name
|
||||
log "No S3 credentials defined. Set WAD_S3_CREDENTIALS= and WAD_S3_BUCKET_NAME= for caching."
|
||||
install
|
||||
elsif get
|
||||
install
|
||||
elsif install
|
||||
put
|
||||
else
|
||||
abort "Failed properly fetch or install. Please review the logs."
|
||||
end
|
||||
end
|
||||
|
||||
def log(message)
|
||||
puts "[wad] #{message}"
|
||||
end
|
||||
end
|
||||
|
||||
if ARGV.index('-v')
|
||||
require 'logger'
|
||||
Presss.logger = Logger.new($stdout)
|
||||
end
|
||||
|
||||
Wad.new.setup
|
||||
__END__
|
||||
Loading…
Reference in New Issue