From e1035840d584e2ebbba52c657e77e1608e95b680 Mon Sep 17 00:00:00 2001 From: Maarten de Waard Date: Tue, 1 Nov 2016 15:38:01 +0100 Subject: [PATCH 1/8] Readme improved --- README.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1f33837..9e840a3 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,11 @@ useful to you in the following cases: .. _Greenhost: https://greenhost.net -Please read the installation instructions on how to configure HAProxy. +This plugin does not configure HAProxy for you, because HAProxy configurations +can can vary a great deal. Please read the installation instructions on how to +configure HAProxy for use with the plugin. If you have a good idea on how we can +implement automatic HAProxy configuration, you are welcome to create a merge +request or an issue. Installing: Requirements ------------------------ From 5a2b97560f65defd247c5f44e56cfb04ed3c0bae Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Dec 2016 15:03:26 +0100 Subject: [PATCH 2/8] Convert MD formatting to RST. --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 9e840a3..045a9cf 100644 --- a/README.rst +++ b/README.rst @@ -455,12 +455,13 @@ Making a `.deb` debian package ------------------------------ Requirements: - - python stdeb: pip install --upgrade stdeb - - dh clean: apt-get install dh-make +- python stdeb: pip install --upgrade stdeb +- dh clean: apt-get install dh-make Run the following commands in your vagrant machine: -``` +.. code:: bash + apt-file update python setup.py sdist # py2dsc has a problem with vbox mounted folders @@ -471,4 +472,4 @@ Run the following commands in your vagrant machine: # NOTE: Not signed, no signed changes (with -uc and -us) # NOTE: Add the package to the ghtools repo dpkg-buildpackage -rfakeroot -uc -us -``` + From 016789cb59165cf72556393425a15573ca422090 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Dec 2016 15:06:07 +0100 Subject: [PATCH 3/8] Formatting/typos fixed in readme. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 045a9cf..f1ffd9d 100644 --- a/README.rst +++ b/README.rst @@ -14,13 +14,13 @@ also enable you to very easily do automatic certificate renewal. Furthermore, you can configure HAProxy to handle Boulder's authentication using the HAProxy authenticator of this plugin. -It was created for use with `Greenhost`_'s share hosting environment and can be +It was created for use with `Greenhost's`_ shared hosting environment and can be useful to you in the following cases: - If you use HAProxy and have several domains for which you want to enable Let's - Encrypt certificates + Encrypt certificates. - If you yourself have a shared hosting platform that uses HAProxy to redirect - to your client's websites + to your client's websites. - Actually any case in which you want to automatically restart HAProxy after you request a new certificate. From 2f027c5a9d6428947d10e4d476124946e42be908 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Dec 2016 15:07:16 +0100 Subject: [PATCH 4/8] Fix link in readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f1ffd9d..1803cef 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ useful to you in the following cases: - Actually any case in which you want to automatically restart HAProxy after you request a new certificate. -.. _Greenhost: https://greenhost.net +.. _Greenhost's: https://greenhost.net This plugin does not configure HAProxy for you, because HAProxy configurations can can vary a great deal. Please read the installation instructions on how to From dc6b3ee0ad131f10a1f39ed6319d4e1adef0d463 Mon Sep 17 00:00:00 2001 From: Maarten de Waard Date: Tue, 13 Dec 2016 18:12:14 +0100 Subject: [PATCH 5/8] fix provisioning for boulder testserver --- README.rst | 21 +++++++++++++++++---- Vagrantfile | 3 +-- greenhost.patch | 40 +++++++++------------------------------- provisioning_server.sh | 2 +- 4 files changed, 28 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index 9e840a3..fbeb080 100644 --- a/README.rst +++ b/README.rst @@ -384,7 +384,19 @@ server, which is the exact same server Let's Encrypt is running. The server is started in Virtual Box using Vagrant. To prevent the installation of any components and dependencies from cluttering up your computer there is also a client Virtual Box instance. Both of these machines can be setup and started by -running the ``dev_start.sh`` script. +running the ``dev_start.sh`` script. This sets up a local boulder server and the +letsencrypt client, so don't worry if it takes more than an hour. + +Vagrant machines +================ +The ``dev_start.sh`` script boots two virtual machines. The first is named +'boulder' and runs a development instance of the boulder server. The second is +'lehaproxy' and runs the client. To test if the machines are setup correctly, +you can SSH into the 'lehaproxy' machine, by running ``vagrant ssh +lehaproxy``. Next, go to the /lehaproxy directory and run +``./tests/boulder-integration.sh``. This runs a modified version of certbot's +boulder-integration test, which tests the HAProxy plugin. If the test succeeds, +your development environment is setup correctly. Development: Running locally without sudo ----------------------------------------- @@ -453,14 +465,15 @@ you can update. Making a `.deb` debian package ------------------------------ + Requirements: - - python stdeb: pip install --upgrade stdeb - - dh clean: apt-get install dh-make +- python stdeb: pip install --upgrade stdeb +- dh clean: apt-get install dh-make Run the following commands in your vagrant machine: -``` +```bash apt-file update python setup.py sdist # py2dsc has a problem with vbox mounted folders diff --git a/Vagrantfile b/Vagrantfile index 2b9ba74..9d6e2f9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -30,7 +30,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| server.vm.box = "debian/jessie64" server.vm.hostname = "boulder.local" server.vm.network :private_network, ip: ENVS['PROJECT_SERVER_IP'] - server.vm.synced_folder ".", "/vagrant/", type: "virtualbox" + server.vm.synced_folder ".", "/boulder/", type: "virtualbox" server.vm.provision "shell" do |s| s.path = './provisioning_server.sh' s.env = ENVS @@ -47,7 +47,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| client.vm.box = "debian/jessie64" client.vm.hostname = PROJECT_NAME + ".local" client.vm.network :private_network, ip: ENVS['PROJECT_CLIENT_IP'] - client.vm.synced_folder ".", "/vagrant/", disabled: true client.vm.synced_folder ".", "/" + PROJECT_NAME + "/", type: "virtualbox" client.vm.provision "shell" do |s| s.path = './provisioning_client.sh' diff --git a/greenhost.patch b/greenhost.patch index aa129de..3602aae 100644 --- a/greenhost.patch +++ b/greenhost.patch @@ -1,10 +1,10 @@ diff --git a/test/config/va.json b/test/config/va.json -index 75ff959..d031f99 100644 +index f3e64ee..1136e98 100644 --- a/test/config/va.json +++ b/test/config/va.json -@@ -3,8 +3,8 @@ +@@ -4,8 +4,8 @@ "userAgent": "boulder", - "debugAddr": "localhost:8004", + "debugAddr": ":8004", "portConfig": { - "httpPort": 5002, - "httpsPort": 5001, @@ -12,31 +12,9 @@ index 75ff959..d031f99 100644 + "httpsPort": 443, "tlsPort": 5001 }, - "maxConcurrentRPCServerRequests": 16, -@@ -37,4 +37,4 @@ - "dnsTimeout": "10s", - "dnsAllowLoopbackAddresses": true - } --} -\ No newline at end of file -+} -diff --git a/test/config/wfe.json b/test/config/wfe.json -index 532da42..29e09fd 100644 ---- a/test/config/wfe.json -+++ b/test/config/wfe.json -@@ -44,7 +44,7 @@ - }, - - "common": { -- "baseURL": "http://127.0.0.1:4000", -+ "baseURL": "http://192.168.33.111:4000", - "issuerCert": "test/test-ca.pem" - } --} -\ No newline at end of file -+} + "lookupIPV6": true, diff --git a/test/rate-limit-policies.yml b/test/rate-limit-policies.yml -index 41aadd3..25d656b 100644 +index 41aadd3..28198b1 100644 --- a/test/rate-limit-policies.yml +++ b/test/rate-limit-policies.yml @@ -4,7 +4,7 @@ totalCertificates: @@ -44,7 +22,7 @@ index 41aadd3..25d656b 100644 certificatesPerName: window: 2160h - threshold: 2 -+ threshold: 10000 ++ threshold: 1000 overrides: ratelimit.me: 1 lim.it: 0 @@ -53,16 +31,16 @@ index 41aadd3..25d656b 100644 pendingAuthorizationsPerAccount: window: 168h # 1 week, should match pending authorization lifetime. - threshold: 3 -+ threshold: 10000 ++ threshold: 1000 certificatesPerFQDNSet: window: 24h - threshold: 5 -+ threshold: 10000 ++ threshold: 1000 overrides: le.wtf: 10000 le1.wtf: 10000 diff --git a/test/test-ca.key-pkcs11.json b/test/test-ca.key-pkcs11.json -index 7a8d348..746dee8 100644 +index b7a44f5..40cc685 100644 --- a/test/test-ca.key-pkcs11.json +++ b/test/test-ca.key-pkcs11.json @@ -1,5 +1,5 @@ diff --git a/provisioning_server.sh b/provisioning_server.sh index 3a4933b..b41be60 100644 --- a/provisioning_server.sh +++ b/provisioning_server.sh @@ -107,7 +107,7 @@ fi # Change pkcs to softhsm and IP to 192.168.33.111 and set high thresholds for rate limiting if grep -Fq "/usr/local/lib/libpkcs11-proxy.so" test/test-ca.key-pkcs11.json; then - git apply /vagrant/greenhost.patch + git apply /boulder/greenhost.patch fi cat < /etc/nginx/sites-available/wfe From 9511e4c1b593bc7428a472fe745c5c1c87e0e4a9 Mon Sep 17 00:00:00 2001 From: Maarten de Waard Date: Mon, 16 Jan 2017 11:03:51 +0100 Subject: [PATCH 6/8] .deb packaging instructions fixed --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 90bc8eb..b362260 100644 --- a/README.rst +++ b/README.rst @@ -474,6 +474,7 @@ Requirements: Run the following commands in your vagrant machine: .. code:: bash + apt-file update python setup.py sdist # py2dsc has a problem with vbox mounted folders From 8a66035004dff748a676650321c9231b0da47418 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 24 Feb 2017 12:32:31 +0100 Subject: [PATCH 7/8] Fix provisioning boulder after changes by the LE team. --- Vagrantfile | 4 +- greenhost.patch | 52 ------------------------ hsmpatch.py | 90 ++++++++++++++++++++++++++++++++++++++++++ provisioning_server.sh | 5 ++- 4 files changed, 95 insertions(+), 56 deletions(-) delete mode 100644 greenhost.patch create mode 100755 hsmpatch.py diff --git a/Vagrantfile b/Vagrantfile index 9d6e2f9..3535b52 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -21,8 +21,8 @@ ENVS = { Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.hostmanager.enabled = true - config.hostmanager.manage_host = true + #config.hostmanager.enabled = true + #config.hostmanager.manage_host = true config.vbguest.auto_update = true config.vbguest.no_remote = false diff --git a/greenhost.patch b/greenhost.patch deleted file mode 100644 index 3602aae..0000000 --- a/greenhost.patch +++ /dev/null @@ -1,52 +0,0 @@ -diff --git a/test/config/va.json b/test/config/va.json -index f3e64ee..1136e98 100644 ---- a/test/config/va.json -+++ b/test/config/va.json -@@ -4,8 +4,8 @@ - "userAgent": "boulder", - "debugAddr": ":8004", - "portConfig": { -- "httpPort": 5002, -- "httpsPort": 5001, -+ "httpPort": 80, -+ "httpsPort": 443, - "tlsPort": 5001 - }, - "lookupIPV6": true, -diff --git a/test/rate-limit-policies.yml b/test/rate-limit-policies.yml -index 41aadd3..28198b1 100644 ---- a/test/rate-limit-policies.yml -+++ b/test/rate-limit-policies.yml -@@ -4,7 +4,7 @@ totalCertificates: - threshold: 100000 - certificatesPerName: - window: 2160h -- threshold: 2 -+ threshold: 1000 - overrides: - ratelimit.me: 1 - lim.it: 0 -@@ -27,10 +27,10 @@ registrationsPerIP: - 127.0.0.1: 1000000 - pendingAuthorizationsPerAccount: - window: 168h # 1 week, should match pending authorization lifetime. -- threshold: 3 -+ threshold: 1000 - certificatesPerFQDNSet: - window: 24h -- threshold: 5 -+ threshold: 1000 - overrides: - le.wtf: 10000 - le1.wtf: 10000 -diff --git a/test/test-ca.key-pkcs11.json b/test/test-ca.key-pkcs11.json -index b7a44f5..40cc685 100644 ---- a/test/test-ca.key-pkcs11.json -+++ b/test/test-ca.key-pkcs11.json -@@ -1,5 +1,5 @@ - { -- "module": "/usr/local/lib/libpkcs11-proxy.so", -+ "module": "/usr/lib/softhsm/libsofthsm.so", - "tokenLabel": "intermediate", - "pin": "5678", - "privateKeyLabel": "intermediate_key" diff --git a/hsmpatch.py b/hsmpatch.py new file mode 100755 index 0000000..a1380c8 --- /dev/null +++ b/hsmpatch.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python2 +""" +Patch the HSM config file to set correct settings for use with a Vagrant +development setup. + +Note: this used to be a simple patch file but since the format changed, it +seems better to parse the file, change the json object and dump it back to the +file. +""" +import simplejson as json +import yaml +import sys +import os.path + +MAX_RECURSION = 100 + +PATCHES = { + "test/config/va.json": { + "va": { + "portConfig": { + "httpPort": 80, + "httpsPort": 443 + } + } + }, + "test/rate-limit-policies.yml": { + "certificatesPerName": { + "threshold": 1000 + }, + "certificatesPerFQDNSet": { + "threshold": 1000 + } + }, + "test/test-ca.key-pkcs11.json": { + "module": "/usr/lib/softhsm/libsofthsm.so", + } +} + + +def recursive_update(old_obj, new_obj, depth=0): + if depth > MAX_RECURSION: + raise RuntimeError("Maximum recursion level reached.") + + if isinstance(new_obj, dict): + for key, value in new_obj.items(): + old_obj[key] = recursive_update( + old_obj[key], new_obj[key], depth+1) + elif isinstance(new_obj, (list, tuple)): + # Merge lists/tuples. + old_obj = old_obj + new_obj + else: + # Set strings, integers, etc. and set() so arrays can be + # overridden. + old_obj = new_obj + return old_obj + + +def patch_yaml(file, obj): + with open(file, "r") as fp: + yaml_obj = yaml.load(fp) + yaml_obj = recursive_update(yaml_obj, obj) + with open(file, "w") as fp: + yaml.dump(yaml_obj, fp, default_flow_style=False) + + +def patch_json(file, obj): + with open(file, "r") as fp: + json_obj = json.load(fp) + json_obj = recursive_update(json_obj, obj) + with open(file, "w") as fp: + json.dump(json_obj, fp, indent=4) + + +if __name__ == '__main__': + try: + for patch_file, patch_obj in PATCHES.items(): + _, file_extension = os.path.splitext(patch_file) + if file_extension in (".yml", ".yaml"): + patch_yaml(patch_file, patch_obj) + elif file_extension in (".json", ".js"): + patch_json(patch_file, patch_obj) + else: + raise NotImplementedError( + "Can't patch files with %s extension" % file_extension) + print("Patched {}".format(os.path.abspath(patch_file))) + + except (OSError, IOError), exc: + print( + "Failed to patch the HSM for development, reason: {}".format(exc)) + sys.exit(1) diff --git a/provisioning_server.sh b/provisioning_server.sh index b41be60..d2f4fa6 100644 --- a/provisioning_server.sh +++ b/provisioning_server.sh @@ -89,7 +89,7 @@ go get bitbucket.org/liamstask/goose/cmd/goose go get -d github.com/letsencrypt/boulder/... # Enter the boulder directory -cd /gopath/src/github.com/letsencrypt/boulder +cd $GOPATH/src/github.com/letsencrypt/boulder # Install alle dependencies godep restore @@ -107,7 +107,8 @@ fi # Change pkcs to softhsm and IP to 192.168.33.111 and set high thresholds for rate limiting if grep -Fq "/usr/local/lib/libpkcs11-proxy.so" test/test-ca.key-pkcs11.json; then - git apply /boulder/greenhost.patch + pip install simplejson pyyaml + /boulder/hsmpatch.py fi cat < /etc/nginx/sites-available/wfe From 825e48f95ccac887fa9db191a149ea510dc6bbf6 Mon Sep 17 00:00:00 2001 From: Maarten de Waard Date: Tue, 15 Aug 2017 17:51:06 +0200 Subject: [PATCH 8/8] bump version number to 0.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e299c63..350df5e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import sys from setuptools import setup from setuptools import find_packages -own_version = '0.1.1' +own_version = '0.1.2' certbot_version = '0.8.1' # Please update tox.ini when modifying dependency version requirements