Merge branch 'release/0.1.2'

README.rst

@@ -14,19 +14,23 @@ also enable you to very easily do automatic certificate renewal.
 Furthermore, you can configure HAProxy to handle Boulder's authentication using
 the HAProxy authenticator of this plugin.
 
-It was created for use with `Greenhost`_'s share hosting environment and can be
+It was created for use with `Greenhost's`_ shared hosting environment and can be
 useful to you in the following cases:
 
 - If you use HAProxy and have several domains for which you want to enable Let's
-  Encrypt certificates
+  Encrypt certificates.
 - If you yourself have a shared hosting platform that uses HAProxy to redirect
-  to your client's websites
+  to your client's websites.
 - Actually any case in which you want to automatically restart HAProxy after you
   request a new certificate.
 
-.. _Greenhost: https://greenhost.net
+.. _Greenhost's: https://greenhost.net
 
-Please read the installation instructions on how to configure HAProxy.
+This plugin does not configure HAProxy for you, because HAProxy configurations
+can can vary a great deal. Please read the installation instructions on how to
+configure HAProxy for use with the plugin. If you have a good idea on how we can
+implement automatic HAProxy configuration, you are welcome to create a merge
+request or an issue.
 
 Installing: Requirements
 ------------------------
@@ -380,7 +384,19 @@ server, which is the exact same server Let's Encrypt is running. The server is
 started in Virtual Box using Vagrant. To prevent the installation of any
 components and dependencies from cluttering up your computer there is also a
 client Virtual Box instance. Both of these machines can be setup and started by
-running the ``dev_start.sh`` script.
+running the ``dev_start.sh`` script. This sets up a local boulder server and the
+letsencrypt client, so don't worry if it takes more than an hour.
+
+Vagrant machines
+================
+The ``dev_start.sh`` script boots two virtual machines. The first is named
+'boulder' and runs a development instance of the boulder server. The second is
+'lehaproxy' and runs the client. To test if the machines are setup correctly,
+you can SSH into the 'lehaproxy' machine, by running ``vagrant ssh
+lehaproxy``. Next, go to the /lehaproxy directory and run
+``./tests/boulder-integration.sh``. This runs a modified version of certbot's
+boulder-integration test, which tests the HAProxy plugin. If the test succeeds,
+your development environment is setup correctly.
 
 Development: Running locally without sudo
 -----------------------------------------
@@ -449,14 +465,16 @@ you can update.
 
 Making a `.deb` debian package
 ------------------------------
+
 Requirements:
 
-    - python stdeb: pip install --upgrade stdeb
+- python stdeb: pip install --upgrade stdeb
-    - dh clean: apt-get install dh-make
+- dh clean: apt-get install dh-make
 
 Run the following commands in your vagrant machine:
 
-```
+.. code:: bash
+
     apt-file update
     python setup.py sdist
     # py2dsc has a problem with vbox mounted folders
@@ -467,4 +485,3 @@ Run the following commands in your vagrant machine:
     # NOTE: Not signed, no signed changes (with -uc and -us)
     # NOTE: Add the package to the ghtools repo
     dpkg-buildpackage -rfakeroot -uc -us
-```

Vagrantfile

@@ -21,8 +21,8 @@ ENVS = {
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-    config.hostmanager.enabled = true
+    #config.hostmanager.enabled = true
-    config.hostmanager.manage_host = true
+    #config.hostmanager.manage_host = true
     config.vbguest.auto_update = true
     config.vbguest.no_remote = false
 
@@ -30,7 +30,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         server.vm.box = "debian/jessie64"
         server.vm.hostname = "boulder.local"
         server.vm.network :private_network, ip:  ENVS['PROJECT_SERVER_IP']
-        server.vm.synced_folder ".", "/vagrant/", type: "virtualbox"
+        server.vm.synced_folder ".", "/boulder/", type: "virtualbox"
         server.vm.provision "shell" do |s|
             s.path = './provisioning_server.sh'
             s.env = ENVS
@@ -47,7 +47,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         client.vm.box = "debian/jessie64"
         client.vm.hostname = PROJECT_NAME + ".local"
         client.vm.network :private_network, ip:  ENVS['PROJECT_CLIENT_IP']
-        client.vm.synced_folder ".", "/vagrant/", disabled: true
         client.vm.synced_folder ".", "/" + PROJECT_NAME + "/", type: "virtualbox"
         client.vm.provision "shell" do |s|
             s.path = './provisioning_client.sh'

greenhost.patch

@@ -1,74 +0,0 @@
-diff --git a/test/config/va.json b/test/config/va.json
-index 75ff959..d031f99 100644
---- a/test/config/va.json
-+++ b/test/config/va.json
-@@ -3,8 +3,8 @@
-     "userAgent": "boulder",
-     "debugAddr": "localhost:8004",
-     "portConfig": {
--      "httpPort": 5002,
--      "httpsPort": 5001,
-+      "httpPort": 80,
-+      "httpsPort": 443,
-       "tlsPort": 5001
-     },
-     "maxConcurrentRPCServerRequests": 16,
-@@ -37,4 +37,4 @@
-     "dnsTimeout": "10s",
-     "dnsAllowLoopbackAddresses": true
-   }
--}
-\ No newline at end of file
-+}
-diff --git a/test/config/wfe.json b/test/config/wfe.json
-index 532da42..29e09fd 100644
---- a/test/config/wfe.json
-+++ b/test/config/wfe.json
-@@ -44,7 +44,7 @@
-   },
- 
-   "common": {
--    "baseURL": "http://127.0.0.1:4000",
-+    "baseURL": "http://192.168.33.111:4000",
-     "issuerCert": "test/test-ca.pem"
-   }
--}
-\ No newline at end of file
-+}
-diff --git a/test/rate-limit-policies.yml b/test/rate-limit-policies.yml
-index 41aadd3..25d656b 100644
---- a/test/rate-limit-policies.yml
-+++ b/test/rate-limit-policies.yml
-@@ -4,7 +4,7 @@ totalCertificates:
-   threshold: 100000
- certificatesPerName:
-   window: 2160h
--  threshold: 2
-+  threshold: 10000
-   overrides:
-     ratelimit.me: 1
-     lim.it: 0
-@@ -27,10 +27,10 @@ registrationsPerIP:
-     127.0.0.1: 1000000
- pendingAuthorizationsPerAccount:
-   window: 168h # 1 week, should match pending authorization lifetime.
--  threshold: 3
-+  threshold: 10000
- certificatesPerFQDNSet:
-   window: 24h
--  threshold: 5
-+  threshold: 10000
-   overrides:
-     le.wtf: 10000
-     le1.wtf: 10000
-diff --git a/test/test-ca.key-pkcs11.json b/test/test-ca.key-pkcs11.json
-index 7a8d348..746dee8 100644
---- a/test/test-ca.key-pkcs11.json
-+++ b/test/test-ca.key-pkcs11.json
-@@ -1,5 +1,5 @@
- {
--    "module": "/usr/local/lib/libpkcs11-proxy.so",
-+    "module": "/usr/lib/softhsm/libsofthsm.so",
-     "tokenLabel": "intermediate",
-     "pin": "5678",
-     "privateKeyLabel": "intermediate_key"

hsmpatch.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python2
+"""
+Patch the HSM config file to set correct settings for use with a Vagrant
+development setup.
+
+Note: this used to be a simple patch file but since the format changed, it
+seems better to parse the file, change the json object and dump it back to the
+file.
+"""
+import simplejson as json
+import yaml
+import sys
+import os.path
+
+MAX_RECURSION = 100
+
+PATCHES = {
+    "test/config/va.json": {
+        "va": {
+            "portConfig": {
+                "httpPort": 80,
+                "httpsPort": 443
+            }
+        }
+    },
+    "test/rate-limit-policies.yml": {
+        "certificatesPerName": {
+            "threshold": 1000
+        },
+        "certificatesPerFQDNSet": {
+            "threshold": 1000
+        }
+    },
+    "test/test-ca.key-pkcs11.json": {
+        "module": "/usr/lib/softhsm/libsofthsm.so",
+    }
+}
+
+
+def recursive_update(old_obj, new_obj, depth=0):
+    if depth > MAX_RECURSION:
+        raise RuntimeError("Maximum recursion level reached.")
+
+    if isinstance(new_obj, dict):
+        for key, value in new_obj.items():
+            old_obj[key] = recursive_update(
+                old_obj[key], new_obj[key], depth+1)
+    elif isinstance(new_obj, (list, tuple)):
+        # Merge lists/tuples.
+        old_obj = old_obj + new_obj
+    else:
+        # Set strings, integers, etc. and set() so arrays can be
+        # overridden.
+        old_obj = new_obj
+    return old_obj
+
+
+def patch_yaml(file, obj):
+    with open(file, "r") as fp:
+        yaml_obj = yaml.load(fp)
+        yaml_obj = recursive_update(yaml_obj, obj)
+    with open(file, "w") as fp:
+        yaml.dump(yaml_obj, fp, default_flow_style=False)
+
+
+def patch_json(file, obj):
+    with open(file, "r") as fp:
+        json_obj = json.load(fp)
+        json_obj = recursive_update(json_obj, obj)
+    with open(file, "w") as fp:
+        json.dump(json_obj, fp, indent=4)
+
+
+if __name__ == '__main__':
+    try:
+        for patch_file, patch_obj in PATCHES.items():
+            _, file_extension = os.path.splitext(patch_file)
+            if file_extension in (".yml", ".yaml"):
+                patch_yaml(patch_file, patch_obj)
+            elif file_extension in (".json", ".js"):
+                patch_json(patch_file, patch_obj)
+            else:
+                raise NotImplementedError(
+                    "Can't patch files with %s extension" % file_extension)
+            print("Patched {}".format(os.path.abspath(patch_file)))
+
+    except (OSError, IOError), exc:
+        print(
+            "Failed to patch the HSM for development, reason: {}".format(exc))
+        sys.exit(1)

provisioning_server.sh

@@ -89,7 +89,7 @@ go get bitbucket.org/liamstask/goose/cmd/goose
 go get -d github.com/letsencrypt/boulder/...
 
 # Enter the boulder directory
-cd /gopath/src/github.com/letsencrypt/boulder
+cd $GOPATH/src/github.com/letsencrypt/boulder
 
 # Install alle dependencies
 godep restore
@@ -107,7 +107,8 @@ fi
 
 # Change pkcs to softhsm and IP to 192.168.33.111 and set high thresholds for rate limiting
 if grep -Fq "/usr/local/lib/libpkcs11-proxy.so" test/test-ca.key-pkcs11.json; then
-    git apply /vagrant/greenhost.patch
+    pip install simplejson pyyaml
+    /boulder/hsmpatch.py
 fi
 
 cat <<EOF > /etc/nginx/sites-available/wfe

setup.py

@@ -3,7 +3,7 @@ import sys
 from setuptools import setup
 from setuptools import find_packages
 
-own_version = '0.1.1'
+own_version = '0.1.2'
 certbot_version = '0.8.1'
 
 # Please update tox.ini when modifying dependency version requirements