From a2ce5c2c305587021152855ae136cf5851401052 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Sun, 12 Jul 2015 13:46:30 -0600 Subject: [PATCH] Enable caching / remove wad bits --- .travis.yml | 17 +-- ci/wad | 334 ---------------------------------------------------- 2 files changed, 6 insertions(+), 345 deletions(-) delete mode 100755 ci/wad diff --git a/.travis.yml b/.travis.yml index 445449d0..319b7994 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,11 @@ addons: packages: - libevent-dev +cache: + directories: + - node_modules + - .pip_download_cache + python: - "2.6" - "2.7" @@ -25,14 +30,6 @@ env: - 'DJANGO="-e git+git://github.com/django/django.git#egg=Django"' global: - 'PIP_DOWNLOAD_CACHE=".pip_download_cache"' - - 'WAD_FILES="package.json,Makefile,setup.py"' - - 'WAD_INSTALL_COMMAND=ci/setup' - - # This should be specified with: - # travis encrypt S3_BUCKET_NAME=secretvalue S3_CREDENTIALS=accesskey:secretkey - # - S3_BUCKET_NAME= - # - S3_CREDENTIALS= - - secure: "QEFKt0HhaMlCW0FCLTz3NDY/P4UuVl1Pw8kSUVKKjbaKn2jZdaLKS68yl3Vyrd9AqrBfRNNLBAnvpZjYL6EwqqnZzpRH3KnAf0WRxE6d5ytrUkwZtOnQaN0Tumuqc3xnoXXalPXvs+abhXZpjfOzx0oPBa2WbtMF/RJZ/bwsTKE=" before_install: # Use closer nameservers @@ -40,11 +37,9 @@ before_install: # These need to be here and not in the env hash because they need to be # evaluated after the virtualenv has been setup - mkdir -p $PIP_DOWNLOAD_CACHE - - 'export WAD_ENVIRONMENT_VARIABLES="TRAVIS_PYTHON_VERSION,TRAVIS_NODE_VERSION,WAD_CACHE_PATH"' - - 'export WAD_CACHE_PATH="node_modules,$PIP_DOWNLOAD_CACHE,$VIRTUAL_ENV"' install: - - time ci/wad + - time ci/setup script: - "if [[ ${TRAVIS_PYTHON_VERSION} != 'pypy' ]]; then make lint; fi" diff --git a/ci/wad b/ci/wad deleted file mode 100755 index 52730008..00000000 --- a/ci/wad +++ /dev/null @@ -1,334 +0,0 @@ -#!/usr/bin/env ruby - -# Generated on: 20-09-2013 at 12:38 - -require 'time' -require 'net/http' -require 'net/https' -require 'digest/md5' -require 'digest/sha1' -require 'fileutils' -require 'openssl' -require 'base64' -require 'cgi' -class Presss - # Computes the Authorization header for a AWS request based on a message, - # the access key ID and secret access key. - class Authorization - attr_accessor :access_key_id, :secret_access_key - - def initialize(access_key_id, secret_access_key) - @access_key_id, @secret_access_key = access_key_id, secret_access_key - end - - # Returns the value for the Authorization header for a message contents. - def header(string) - 'AWS ' + access_key_id + ':' + sign(string) - end - - # Returns a signature for a AWS request message. - def sign(string) - Base64.encode64(hmac_sha1(string)).strip - end - - def hmac_sha1(string) - OpenSSL::HMAC.digest('sha1', secret_access_key, string) - end - end - - class HTTP - attr_accessor :config - - def initialize(config) - @config = config - end - - # Returns the configured bucket name. - def bucket_name - config[:bucket_name] - end - - def region - config[:region] || 'us-east-1' - end - - def domain - case region - when 'us-east-1' - 's3.amazonaws.com' - else - 's3-%s.amazonaws.com' % region - end - end - - def bucket_in_hostname? - config[:bucket_in_hostname] - end - - def url_prefix - if bucket_in_hostname? - "https://#{bucket_name}.#{domain}" - else - "https://#{domain}/#{bucket_name}" - end - end - - # Returns the absolute path based on the key for the object. - def absolute_path(path) - path.start_with?('/') ? path : '/' + path - end - - # Returns the canonicalized resource used in the authorization - # signature for an absolute path to an object. - def canonicalized_resource(path) - if bucket_name.nil? - raise ArgumentError, "Please configure a bucket_name: Presss.config = { bucket_name: 'my-bucket-name }" - else - '/' + bucket_name + absolute_path(path) - end - end - - # Returns a Presss::Authorization instance for the configured - # AWS credentials. - def authorization - @authorization ||= Presss::Authorization.new( - config[:access_key_id], - config[:secret_access_key] - ) - end - - def signed_url(verb, expires, headers, path) - path = absolute_path(path) - canonical_path = canonicalized_resource(path) - signature = [ verb.to_s.upcase, nil, nil, expires, [ headers, canonical_path ].flatten.compact ].flatten.join("\n") - signed = authorization.sign(signature) - "#{url_prefix}#{path}?Signature=#{CGI.escape(signed)}&Expires=#{expires}&AWSAccessKeyId=#{CGI.escape(authorization.access_key_id)}" - end - - def download(path, destination) - url = signed_url(:get, Time.now.to_i + 600, nil, path) - Presss.log "signed_url=#{url}" - system 'curl', '-f', '-S', '-o', destination, url - $?.success? - end - - # Puts an object with a key using a file or string. Optionally pass in - # the content-type if you want to set a specific one. - def put(path, file) - header = 'x-amz-storage-class:REDUCED_REDUNDANCY' - url = signed_url(:put, Time.now.to_i + 600, header, path) - Presss.log "signed_url=#{url}" - system 'curl', '-f', '-S', '-H', header, '-T', file, url - $?.success? - end - end - - class << self - attr_accessor :config - attr_accessor :logger - end - self.config = {} - - # Get a object with a certain key. - def self.download(path, destination) - t0 = Time.now - request = Presss::HTTP.new(config) - log("Trying to GET #{path}") - if request.download(path, destination) - puts("[wad] Downloaded in #{(Time.now - t0).to_i} seconds") - true - else - nil - end - end - - # Puts an object with a key using a file or string. Optionally pass in - # the content-type if you want to set a specific one. - def self.put(path, filename, content_type='application/x-download') - request = Presss::HTTP.new(config) - log("Trying to PUT #{path}") - request.put(path, filename) - end - - # Logs to the configured logger if a logger was configured. - def self.log(message) - if logger - logger.info('[Presss] ' + message) - end - end -end - -# Utility class to push and fetch Bundler directories to speed up -# test runs on Travis-CI -class Wad - class Key - def default_environment_variables - [] - end - - def default_files - [ "#{ENV['BUNDLE_GEMFILE']}.lock" ] - end - - def environment_variables - if ENV['WAD_ENVIRONMENT_VARIABLES'] - ENV['WAD_ENVIRONMENT_VARIABLES'].split(',') - else - default_environment_variables - end - end - - def files - ENV['WAD_FILES'] ? ENV['WAD_FILES'].split(',') : default_files - end - - def environment_variable_contents - environment_variables.map { |v| ENV[v] } - end - - def file_contents - files.map { |f| File.read(f) rescue nil } - end - - def contents - segments = [ RUBY_VERSION, RUBY_PLATFORM ] + environment_variable_contents + file_contents - Digest::SHA1.hexdigest(segments.join("\n")) - end - end - - def initialize - s3_configure - end - - def project_root - Dir.pwd - end - - def artifact_name - @artifact_name ||= Key.new.contents - end - - def bzip_filename - File.join(project_root, "tmp/#{artifact_name}.tar.bz2") - end - - def cache_path - ENV['WAD_CACHE_PATH'] ? ENV['WAD_CACHE_PATH'].split(",") : [ '.bundle' ] - end - - def s3_bucket_name - if bucket = ENV['WAD_S3_BUCKET_NAME'] || ENV['S3_BUCKET_NAME'] - bucket - end - end - - def s3_credentials - if creds = ENV['WAD_S3_CREDENTIALS'] || ENV['S3_CREDENTIALS'] - creds.split(':') - end - end - - def s3_access_key_id - s3_credentials && s3_credentials[0] - end - - def s3_secret_access_key - s3_credentials && s3_credentials[1] - end - - def s3_path - "#{artifact_name}.tar.bz2" - end - - def s3_configure - Presss.config = { - :bucket_name => s3_bucket_name, - :access_key_id => s3_access_key_id, - :secret_access_key => s3_secret_access_key, - :region => ENV['WAD_AWS_REGION'], - :bucket_in_hostname => (ENV['WAD_BUCKET_IN_HOSTNAME'] == 'true') - } - end - - def s3_write - log "Trying to write Wad to S3" - if Presss.put(s3_path, bzip_filename) - log "Wrote Wad to S3" - else - log "Failed to write to S3, debug with `wad -v'" - end - end - - def s3_read - if File.exist?(bzip_filename) - log "Removing bundle from filesystem" - FileUtils.rm_f(bzip_filename) - end - - log "Trying to fetch Wad from S3" - FileUtils.mkdir_p(File.dirname(bzip_filename)) - Presss.download(s3_path, bzip_filename) - end - - def zip - log "Creating artifact with tar (#{File.basename(bzip_filename)})" - system("cd #{project_root} && tar -cPjf #{bzip_filename} #{cache_path.join(' ')}") - $?.success? - end - - - def unzip - log "Unpacking artifact with tar (#{File.basename(bzip_filename)})" - system("cd #{project_root} && tar -xPjf #{bzip_filename}") - $?.success? - end - - def put - zip - s3_write - end - - def get - if s3_read - unzip - end - end - - def default_command - bundle_without = ENV['WAD_BUNDLE_WITHOUT'] || "development production" - "bundle install --path .bundle --without='#{bundle_without}'" - end - - def install - log "Installing..." - command = ENV['WAD_INSTALL_COMMAND'] || default_command - puts command - system(command) - $?.success? - end - - def setup - if !s3_credentials || !s3_bucket_name - log "No S3 credentials defined. Set WAD_S3_CREDENTIALS= and WAD_S3_BUCKET_NAME= for caching." - install - elsif get - install - elsif install - put - else - abort "Failed properly fetch or install. Please review the logs." - end - end - - def log(message) - puts "[wad] #{message}" - end -end - -if ARGV.index('-v') - require 'logger' - Presss.logger = Logger.new($stdout) -end - -Wad.new.setup -__END__