commit 4e27b279c209b5b8977de70e21c8f0e080e9d444 Author: Vincent Fretin Date: Tue Dec 11 18:09:50 2012 +0100 Added initial buildout structure diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 0000000..98e17f0 --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,262 @@ +############################################################################## +# +# Copyright (c) 2006 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Bootstrap a buildout-based project + +Simply run this script in a directory containing a buildout.cfg. +The script accepts buildout command-line options, so you can +use the -c option to specify an alternate configuration file. +""" + +import os, shutil, sys, tempfile, urllib, urllib2, subprocess +from optparse import OptionParser + +if sys.platform == 'win32': + def quote(c): + if ' ' in c: + return '"%s"' % c # work around spawn lamosity on windows + else: + return c +else: + quote = str + +# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments. +stdout, stderr = subprocess.Popen( + [sys.executable, '-Sc', + 'try:\n' + ' import ConfigParser\n' + 'except ImportError:\n' + ' print 1\n' + 'else:\n' + ' print 0\n'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +has_broken_dash_S = bool(int(stdout.strip())) + +# In order to be more robust in the face of system Pythons, we want to +# run without site-packages loaded. This is somewhat tricky, in +# particular because Python 2.6's distutils imports site, so starting +# with the -S flag is not sufficient. However, we'll start with that: +if not has_broken_dash_S and 'site' in sys.modules: + # We will restart with python -S. + args = sys.argv[:] + args[0:0] = [sys.executable, '-S'] + args = map(quote, args) + os.execv(sys.executable, args) +# Now we are running with -S. We'll get the clean sys.path, import site +# because distutils will do it later, and then reset the path and clean +# out any namespace packages from site-packages that might have been +# loaded by .pth files. +clean_path = sys.path[:] +import site # imported because of its side effects +sys.path[:] = clean_path +for k, v in sys.modules.items(): + if k in ('setuptools', 'pkg_resources') or ( + hasattr(v, '__path__') and + len(v.__path__) == 1 and + not os.path.exists(os.path.join(v.__path__[0], '__init__.py'))): + # This is a namespace package. Remove it. + sys.modules.pop(k) + +is_jython = sys.platform.startswith('java') + +setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py' +distribute_source = 'http://python-distribute.org/distribute_setup.py' + + +# parsing arguments +def normalize_to_url(option, opt_str, value, parser): + if value: + if '://' not in value: # It doesn't smell like a URL. + value = 'file://%s' % ( + urllib.pathname2url( + os.path.abspath(os.path.expanduser(value))),) + if opt_str == '--download-base' and not value.endswith('/'): + # Download base needs a trailing slash to make the world happy. + value += '/' + else: + value = None + name = opt_str[2:].replace('-', '_') + setattr(parser.values, name, value) + +usage = '''\ +[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] + +Bootstraps a buildout-based project. + +Simply run this script in a directory containing a buildout.cfg, using the +Python that you want bin/buildout to use. + +Note that by using --setup-source and --download-base to point to +local resources, you can keep this script from going over the network. +''' + +parser = OptionParser(usage=usage) +parser.add_option("-v", "--version", dest="version", + help="use a specific zc.buildout version") +parser.add_option("-d", "--distribute", + action="store_true", dest="use_distribute", default=True, + help="Use Distribute rather than Setuptools.") +parser.add_option("--setup-source", action="callback", dest="setup_source", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or file location for the setup file. " + "If you use Setuptools, this will default to " + + setuptools_source + "; if you use Distribute, this " + "will default to " + distribute_source + ".")) +parser.add_option("--download-base", action="callback", dest="download_base", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or directory for downloading " + "zc.buildout and either Setuptools or Distribute. " + "Defaults to PyPI.")) +parser.add_option("--eggs", + help=("Specify a directory for storing eggs. Defaults to " + "a temporary directory that is deleted when the " + "bootstrap script completes.")) +parser.add_option("-t", "--accept-buildout-test-releases", + dest='accept_buildout_test_releases', + action="store_true", default=False, + help=("Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas.")) +parser.add_option("-c", None, action="store", dest="config_file", + help=("Specify the path to the buildout configuration " + "file to be used.")) + +options, args = parser.parse_args() + +# if -c was provided, we push it back into args for buildout's main function +if options.config_file is not None: + args += ['-c', options.config_file] + +if options.eggs: + eggs_dir = os.path.abspath(os.path.expanduser(options.eggs)) +else: + eggs_dir = tempfile.mkdtemp() + +if options.setup_source is None: + if options.use_distribute: + options.setup_source = distribute_source + else: + options.setup_source = setuptools_source + +if options.accept_buildout_test_releases: + args.append('buildout:accept-buildout-test-releases=true') +args.append('bootstrap') + +try: + import pkg_resources + import setuptools # A flag. Sometimes pkg_resources is installed alone. + if not hasattr(pkg_resources, '_distribute'): + raise ImportError +except ImportError: + ez_code = urllib2.urlopen( + options.setup_source).read().replace('\r\n', '\n') + ez = {} + exec ez_code in ez + setup_args = dict(to_dir=eggs_dir, download_delay=0) + if options.download_base: + setup_args['download_base'] = options.download_base + if options.use_distribute: + setup_args['no_fake'] = True + ez['use_setuptools'](**setup_args) + if 'pkg_resources' in sys.modules: + reload(sys.modules['pkg_resources']) + import pkg_resources + # This does not (always?) update the default working set. We will + # do it. + for path in sys.path: + if path not in pkg_resources.working_set.entries: + pkg_resources.working_set.add_entry(path) + +cmd = [quote(sys.executable), + '-c', + quote('from setuptools.command.easy_install import main; main()'), + '-mqNxd', + quote(eggs_dir)] + +if not has_broken_dash_S: + cmd.insert(1, '-S') + +find_links = options.download_base +if not find_links: + find_links = os.environ.get('bootstrap-testing-find-links') +if find_links: + cmd.extend(['-f', quote(find_links)]) + +if options.use_distribute: + setup_requirement = 'distribute' +else: + setup_requirement = 'setuptools' +ws = pkg_resources.working_set +setup_requirement_path = ws.find( + pkg_resources.Requirement.parse(setup_requirement)).location +env = dict( + os.environ, + PYTHONPATH=setup_requirement_path) + +requirement = 'zc.buildout' +version = options.version +if version is None and not options.accept_buildout_test_releases: + # Figure out the most recent final version of zc.buildout. + import setuptools.package_index + _final_parts = '*final-', '*final' + + def _final_version(parsed_version): + for part in parsed_version: + if (part[:1] == '*') and (part not in _final_parts): + return False + return True + index = setuptools.package_index.PackageIndex( + search_path=[setup_requirement_path]) + if find_links: + index.add_find_links((find_links,)) + req = pkg_resources.Requirement.parse(requirement) + if index.obtain(req) is not None: + best = [] + bestv = None + for dist in index[req.project_name]: + distv = dist.parsed_version + if _final_version(distv): + if bestv is None or distv > bestv: + best = [dist] + bestv = distv + elif distv == bestv: + best.append(dist) + if best: + best.sort() + version = best[-1].version +if version: + requirement = '=='.join((requirement, version)) +cmd.append(requirement) + +if is_jython: + import subprocess + exitcode = subprocess.Popen(cmd, env=env).wait() +else: # Windows prefers this, apparently; otherwise we would prefer subprocess + exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env])) +if exitcode != 0: + sys.stdout.flush() + sys.stderr.flush() + print ("An error occurred when trying to install zc.buildout. " + "Look above this message for any errors that " + "were output by easy_install.") + sys.exit(exitcode) + +ws.add_entry(eggs_dir) +ws.require(requirement) +import zc.buildout.buildout +zc.buildout.buildout.main(args) +if not options.eggs: # clean up temporary egg directory + shutil.rmtree(eggs_dir) diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 0000000..9373c3f --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,91 @@ +[v] +policy-package = +port-base = 80 + +[buildout] +extends = + http://dist.plone.org/release/4.2.2/versions.cfg + sources.cfg + checkouts.cfg + versions.cfg + tests.cfg +parts += + instance1 + zopepy + backup + omelette +extensions = + mr.developer + buildout.dumppickedversions +versions = versions +allow-hosts = + pypi.python.org + pypi.ecreall.com + effbot.org + +find-links = + http://dist.plone.org/release/4.2.2/ + http://pypi.ecreall.com/pypi/simple + +eggs += + Pillow + +[instance1] +recipe = plone.recipe.zope2instance +user = admin:KevakIlhut4 +eggs = + ${buildout:eggs} + ${v:policy-package} +zcml = ${v:policy-package} +environment-vars = + PYTHON_EGG_CACHE ${buildout:directory}/tmp + TMPDIR ${buildout:directory}/tmp + zope_i18n_compile_mo_files true +event-log-custom = + + level info + path ${buildout:directory}/var/log/${:_buildout_section_name_}.log + max-size 1MB + old-files 720 + + + level error + path ${buildout:directory}/var/log/error.log + max-size 1MB + old-files 720 + +access-log-custom = + + path ${buildout:directory}/var/log/${:_buildout_section_name_}-Z2.log + max-size 1MB + old-files 720 + +mailinglogger = + + level error + flood-level 10 +# smtp-server localhost:9025 + from development@ecreall.com + to development@ecreall.com + subject [Error on Project X] [%(hostname)s] %(line)s + + +[zopepy] +recipe = zc.recipe.egg +eggs = ${instance1:eggs} +interpreter = zopepy +scripts = zopepy + +[backup] +recipe = collective.recipe.backup +keep = 2 +keep_blob_days = 14 +full = false +gzip = true +blob_storage = ${buildout:directory}/var/blobstorage + +[omelette] +recipe = collective.recipe.omelette +eggs = + ${instance1:eggs} + ${test:eggs} diff --git a/checkouts.cfg b/checkouts.cfg new file mode 100644 index 0000000..d5798b1 --- /dev/null +++ b/checkouts.cfg @@ -0,0 +1,3 @@ +[buildout] +always-checkout = force +auto-checkout += diff --git a/deployment.cfg b/deployment.cfg new file mode 100644 index 0000000..4af1eaa --- /dev/null +++ b/deployment.cfg @@ -0,0 +1,28 @@ +[buildout] +extends-cache = extends-cache +extends = staging.cfg +parts += + backup-template + backup-schedule + zeopack-schedule + +[backup-template] +recipe = collective.recipe.template +inline = + #!/bin/bash + ${buildout:bin-directory}/backup -q + rsync -a --delete ${backup:location}/ sav@host:/home/sav/projectx/backups/ + rsync -aH --delete ${backup:blobbackuplocation}/ sav@host:/home/sav/projextx/blobstoragebackups/ +output = ${buildout:bin-directory}/backup.sh +mode = 755 + +[backup-schedule] +recipe = z3c.recipe.usercrontab +times = 0 4 * * * +command = ${backup-template:output} + +# every sunday +[zeopack-schedule] +recipe = z3c.recipe.usercrontab +times = 0 1 * * 0 +command = ${buildout:bin-directory}/zeopack -B ${buildout:directory}/var/blobstorage/ diff --git a/development.cfg b/development.cfg new file mode 100644 index 0000000..cb81b55 --- /dev/null +++ b/development.cfg @@ -0,0 +1,30 @@ +[buildout] +extends-cache = +extends = + staging.cfg + conf.ini +parts += + checkversions + +[instance1] +http-address = ${ports:instance1} +mailinglogger = +debug-mode = on +verbose-security = on +eggs += + plone.reload + Products.PDBDebugMode + Products.DocFinderTab + aws.zope2zcmldoc + collective.profiler +zcml += + plone.reload +zope-conf-additional = + publisher-profile-file ${buildout:directory}/profile.dat + +[supervisor] +eventlisteners = + +[checkversions] +recipe = zc.recipe.egg +eggs = z3c.checkversions [buildout] diff --git a/eclipse.cfg b/eclipse.cfg new file mode 100644 index 0000000..68fe403 --- /dev/null +++ b/eclipse.cfg @@ -0,0 +1,3 @@ +[buildout] +extends = development.cfg +always-checkout = false diff --git a/sources.cfg b/sources.cfg new file mode 100644 index 0000000..d64526e --- /dev/null +++ b/sources.cfg @@ -0,0 +1,11 @@ +[buildout] +extends = + https://raw.github.com/plone/buildout.coredev/4.2/sources.cfg + +[remotes] +tdesvenain = git://github.com/tdesvenain +tdesvenain_push = git@github.com:tdesvenain + +[sources] +ecreall.helpers.testing = git ${remotes:tdesvenain}/ecreall.helpers.testing.git pushurl=${remotes:tdesvenain_push}/ecreall.helpers.testing.git +ecreall.helpers.upgrade = git ${remotes:tdesvenain}/ecreall.helpers.upgrade.git pushurl=${remotes:tdesvenain_push}/ecreall.helpers.upgrade.git diff --git a/staging.cfg b/staging.cfg new file mode 100644 index 0000000..12806eb --- /dev/null +++ b/staging.cfg @@ -0,0 +1,130 @@ +[buildout] +extends-cache = extends-cache +extends = + buildout.cfg +parts += + zeoserver + instance1 + instance2 + instance3 + instance-debug + varnish-build + varnish-conf + varnish + haproxy + haproxy-conf + supervisor + +[hosts] +zeoserver = 127.0.0.1 +instance1 = 127.0.0.1 +instance2 = 127.0.0.1 +instance3 = 127.0.0.1 +instance-debug = 127.0.0.1 +varnish = 127.0.0.1 +haproxy = 127.0.0.1 +supervisor = 127.0.0.1 + +[ports] +zeoserver = ${v:port-base}00 +instance1 = ${v:port-base}81 +instance2 = ${v:port-base}82 +instance3 = ${v:port-base}83 +instance-debug = ${v:port-base}80 +varnish = ${v:port-base}50 +haproxy = ${v:port-base}55 +supervisor = ${v:port-base}91 + +[users] +zope = zope +varnish = zope +haproxy = zope + +[zeoserver] +recipe = plone.recipe.zeoserver +zeo-address = ${hosts:zeoserver}:${ports:zeoserver} + +[instance1] +shared-blob = on +zeo-client = true +zeo-address = ${zeoserver:zeo-address} +effective-user = ${users:zope} +http-address = ${hosts:instance1}:${ports:instance1} + +[instance2] +<= instance1 +http-address = ${hosts:instance2}:${ports:instance2} + +[instance3] +<= instance1 +http-address = ${hosts:instance3}:${ports:instance3} + +[instance-debug] +<= instance1 +http-address = ${hosts:instance-debug}:${ports:instance-debug} +debug-mode = on +verbose-security = on +eggs = + ${instance1:eggs} + Products.LongRequestLogger [standalone] +environment-vars = + ${instance1:environment-vars} + longrequestlogger_file ${buildout:directory}/var/log/longrequest.log + longrequestlogger_timeout 2 + longrequestlogger_interval 2 + +[supervisor] +recipe = collective.recipe.supervisor +port = ${ports:supervisor} +user = admin +password = admin +plugins = superlance +supervisord-conf = ${buildout:directory}/etc/supervisord.conf +serverurl = http://${hosts:supervisor}:${ports:supervisor} +programs = + 10 zeoserver ${zeoserver:location}/bin/runzeo ${zeoserver:location} true ${users:zope} + 20 instance1 ${buildout:bin-directory}/instance1 [console] ${instance1:location} true + 30 instance2 ${buildout:bin-directory}/instance2 [console] ${instance2:location} true + 40 instance3 ${buildout:bin-directory}/instance3 [console] ${instance3:location} true + 60 haproxy ${buildout:bin-directory}/haproxy [-f ${buildout:directory}/etc/haproxy.conf -db] true + 70 varnish (autorestart=true) ${buildout:bin-directory}/varnish ${varnish:location} true +eventlisteners = + memmon TICK_60 ${buildout:bin-directory}/memmon [-p instance1=1200MB -p instance2=1200MB -m crt-monitoring@ecreall.com] + crashmail PROCESS_STATE ${buildout:bin-directory}/crashmail [-m vincentfretin@ecreall.com] + +[varnish-build] +recipe = zc.recipe.cmmi +url = ${varnish:download-url} + +[varnish-conf] +recipe = collective.recipe.template +input = ${buildout:directory}/templates/varnish.vcl.in +output = ${buildout:directory}/etc/varnish.vcl + +[varnish] +recipe = plone.recipe.varnish +#download-url = ${versions:varnish} +daemon = ${buildout:parts-directory}/varnish-build/sbin/varnishd +bind = ${hosts:varnish}:${ports:varnish} +#backends = ${haproxy-conf:bind} +config = ${varnish-conf:output} +cache-size = 256M +user = ${users:varnish} +mode = foreground + +[haproxy] +recipe = plone.recipe.haproxy +#url = ${versions:haproxy-url} +#cpu = i686 +target = linux26 +pcre = 1 + +[haproxy-conf] +recipe = collective.recipe.template +input = ${buildout:directory}/templates/haproxy.conf.in +output = ${buildout:directory}/etc/haproxy.conf +maxconn = 32000 +ulimit-n = 65536 +user = ${users:haproxy} +group = ${users:haproxy} +bind = ${hosts:haproxy}:${ports:haproxy} diff --git a/templates/conf.ini b/templates/conf.ini new file mode 100644 index 0000000..7d13e42 --- /dev/null +++ b/templates/conf.ini @@ -0,0 +1,4 @@ +[users] +zope = vincentfretin +varnish = vincentfretin +haproxy = vincentfretin \ No newline at end of file diff --git a/templates/haproxy.conf.in b/templates/haproxy.conf.in new file mode 100644 index 0000000..6898587 --- /dev/null +++ b/templates/haproxy.conf.in @@ -0,0 +1,53 @@ +global + log 127.0.0.1 local6 + maxconn ${haproxy-conf:maxconn} + ulimit-n ${haproxy-conf:ulimit-n} + user ${haproxy-conf:user} + group ${haproxy-conf:group} + daemon + nbproc 1 + +defaults +# log global + mode http + + # The zope instances have maxconn 1, and it is important that + # unused/idle connections are closed as soon as possible. + option httpclose + + # Remove requests from the queue if people press stop button + option abortonclose + +# option httplog +# option dontlognull + retries 3 + option redispatch + monitor-uri /haproxy-ping + + timeout connect 7s + timeout queue 300s + timeout client 300s + timeout server 300s + + stats enable + stats uri /haproxy-status + stats refresh 5s + stats realm Haproxy\ statistics + +frontend zopecluster + bind ${haproxy-conf:bind} + default_backend zope + + # capture cookie __ac len 10 + # option httplog + # log 127.0.0.1 local6 + +# Load balancing over the zope instances +backend zope + balance roundrobin + cookie serverid insert nocache indirect + option httpchk GET / + + server plone0101 ${hosts:instance1}:${ports:instance1} cookie p0101 maxconn 2 check inter 2000 rise 2 fall 5 + server plone0102 ${instance2:http-address} cookie p0102 maxconn 2 check inter 2000 rise 2 fall 5 + server plone0102 ${instance3:http-address} cookie p0103 maxconn 2 check inter 2000 rise 2 fall 5 diff --git a/templates/varnish.vcl.in b/templates/varnish.vcl.in new file mode 100644 index 0000000..f39c064 --- /dev/null +++ b/templates/varnish.vcl.in @@ -0,0 +1,122 @@ +# This is a basic VCL configuration file for varnish. See the vcl(7) +# man page for details on VCL syntax and semantics. + +backend backend_0 { + .host = "${hosts:haproxy}"; + .port = "${ports:haproxy}"; + .connect_timeout = 0.4s; + .first_byte_timeout = 300s; + .between_bytes_timeout = 60s; +} + + +acl purge { + "localhost"; + "127.0.0.1"; +} + +sub vcl_recv { + set req.grace = 120s; + set req.backend = backend_0; + + if (req.request == "PURGE") { + if (!client.ip ~ purge) { + error 405 "Not allowed."; + } + return(lookup); + } + + if (req.request != "GET" && + req.request != "HEAD" && + req.request != "PUT" && + req.request != "POST" && + req.request != "TRACE" && + req.request != "OPTIONS" && + req.request != "DELETE") { + /* Non-RFC2616 or CONNECT which is weird. */ + return(pipe); + } + + if (req.request != "GET" && req.request != "HEAD") { + /* We only deal with GET and HEAD by default */ + return(pass); + } + + if (req.http.If-None-Match) { + return(pass); + } + + if (req.url ~ "createObject") { + return(pass); + } + + if (req.url ~ "openTrashcan") { + return(pass); + } + + if (req.url ~ "closeTrashcan") { + return(pass); + } + + if (req.url ~ "subsite-skin") { + return(pass); + } + + if (req.url ~ "solgemafullcalendar_vars.js") { + return(pass); + } + + if (req.url ~ "updated_search") { + return(pass); + } + + remove req.http.Accept-Encoding; + + return(lookup); +} + +sub vcl_pipe { + # This is not necessary if you do not do any request rewriting. + set req.http.connection = "close"; +} + +sub vcl_hit { + if (req.request == "PURGE") { + purge_url(req.url); + error 200 "Purged"; + } + + if (!obj.cacheable) { + return(pass); + } +} + +sub vcl_miss { + if (req.request == "PURGE") { + error 404 "Not in cache"; + } + +} + +sub vcl_fetch { + set beresp.grace = 120s; + if (!beresp.cacheable) { + return(pass); + } + if (beresp.http.Set-Cookie) { + return(pass); + } + if (beresp.http.Cache-Control ~ "(private|no-cache|no-store)") { + return(pass); + } + if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") { + return(pass); + } + if (beresp.http.Content-Type == "application/x-zope-edit") { + return(pass); + } + if (beresp.http.url ~ "subsite-skin") { + return(pass); + } +} + diff --git a/tests.cfg b/tests.cfg new file mode 100644 index 0000000..8a0bc68 --- /dev/null +++ b/tests.cfg @@ -0,0 +1,30 @@ +[buildout] +parts += + test + test-ptc + coverage-test + coverage-report + +[test] +recipe = zc.recipe.testrunner +eggs = + ${buildout:eggs} +defaults = ['--auto-color', '--auto-progress'] + +[test-ptc] +recipe = zc.recipe.testrunner +eggs = + ${buildout:eggs} + Products.PloneTestCase +defaults = ['--auto-color', '--auto-progress'] + +[coverage-test] +recipe = zc.recipe.testrunner +eggs = ${test:eggs} +defaults = ['--coverage', '../../coverage', '-v', '--auto-progress'] + +[coverage-report] +recipe = zc.recipe.egg +eggs = z3c.coverage +scripts = coveragereport +arguments = ('coverage', 'report') diff --git a/versions.cfg b/versions.cfg new file mode 100644 index 0000000..8333480 --- /dev/null +++ b/versions.cfg @@ -0,0 +1 @@ +[versions]