Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Nixos #1

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
Open

Nixos #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
90f3ce5
Nixos: Added environment and packages.
reinhardt Nov 6, 2019
e14a90a
Upgraded batou
reinhardt Nov 6, 2019
1370f64
Added batou_ext via requirements.txt
reinhardt Nov 6, 2019
d44b907
production-nixos: Removed target_directory
reinhardt Nov 6, 2019
07b38ea
Nixos: Moved java to the solr component.
reinhardt Nov 6, 2019
9101887
Use fcio.net domain
reinhardt Jan 24, 2020
cd1b1cf
Black
reinhardt Feb 28, 2020
2ccdee9
Merge branch 'master' into nixos
reinhardt Mar 4, 2020
9088a4b
Also install nix packages on zeo host.
reinhardt Apr 23, 2020
4ad8d46
Copied secrets from production to production-nixos
reinhardt May 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion batou
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import subprocess
# download and copy this file to bootstrap a new batou installation.
BOOTSTRAP_VERSION_MARKER = '{' + '{version}' + '}'

version = os.environ.get('BATOU_VERSION', '1.7.5')
version = os.environ.get('BATOU_VERSION', '1.11.0')
develop = os.environ.get('BATOU_DEVELOP', '')


Expand Down
71 changes: 39 additions & 32 deletions components/solr/component.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,63 +3,70 @@
from batou.lib.file import Directory
from batou.lib.supervisor import Program
from batou.utils import Address
import batou_ext.nix


class Solr4(Component):

address = Attribute(Address, '{{host.fqdn}}:8984')
test_address = Attribute(Address, '{{host.fqdn}}:9984')
address = Attribute(Address, "{{host.fqdn}}:8984")
test_address = Attribute(Address, "{{host.fqdn}}:9984")

profile = 'base'
profile = "base"

def configure(self):
self.solr_version = '4.9.1'
self.solr_version = "4.9.1"
self.major_version = self.solr_version[0]
self.solr_src = (
'http://archive.apache.org/dist/lucene/solr/'
'{0}/solr-{0}.zip'.format(
self.solr_version
)
"http://archive.apache.org/dist/lucene/solr/"
"{0}/solr-{0}.zip".format(self.solr_version)
)
self.md5sum = '2b8db289ce238c665363017f62e54708'
self.md5sum = "2b8db289ce238c665363017f62e54708"

self += Buildout(
python='2.7',
setuptools='38.5.1',
version='2.11.4',
additional_config=[Directory('profiles', source='profiles')])
python="2.7",
setuptools="38.5.1",
version="2.11.4",
additional_config=[Directory("profiles", source="profiles")],
)


class Solr5(Component):

address = Attribute(Address, '{{host.fqdn}}:8985')
test_address = Attribute(Address, '{{host.fqdn}}:9985')
address = Attribute(Address, "{{host.fqdn}}:8985")
test_address = Attribute(Address, "{{host.fqdn}}:9985")

profile = 'solr5'
profile = "solr5"

def configure(self):
self.provide('solr:server', self)
self.solr_version = '5.5.5'
self.provide("solr:server", self)
self.solr_version = "5.5.5"
self.major_version = self.solr_version[0]
self.solr_src = (
'http://archive.apache.org/dist/lucene/solr/'
'{0}/solr-{0}.zip'.format(
self.solr_version
)
"http://archive.apache.org/dist/lucene/solr/"
"{0}/solr-{0}.zip".format(self.solr_version)
)
self.md5sum = 'b15105d42936f33de3e5d241c6e6cf71'
self.md5sum = "b15105d42936f33de3e5d241c6e6cf71"

self += batou_ext.nix.UserEnv(
"solr",
packages=["jre8"],
channel="https://releases.nixos.org/nixos/19.03/nixos-19.03.173575.0e0ee084d6d/nixexprs.tar.xz",
)
self += Buildout(
python='2.7',
setuptools='38.5.1',
version='2.11.4',
python="2.7",
setuptools="38.5.1",
version="2.11.4",
additional_config=[
Directory('profiles', source='profiles'),
Directory('templates', source='templates'),
Directory("profiles", source="profiles"),
Directory("templates", source="templates"),
],
)

self += Program('solr',
command='/usr/bin/env java',
args=self.expand('-Xms512m -Xmx2048m -jar start.jar --module=http jetty.host={{component.address.connect.host}} jetty.port={{component.address.connect.port}}'),
directory=self.map('parts/instance'))
self += Program(
"solr",
command="/usr/bin/env java",
args=self.expand(
"-Xms512m -Xmx2048m -jar start.jar --module=http jetty.host={{component.address.connect.host}} jetty.port={{component.address.connect.port}}"
),
directory=self.map("parts/instance"),
)
36 changes: 36 additions & 0 deletions components/zeo/component.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from batou.lib.git import Clone
from batou.lib.supervisor import Program
from batou.utils import Address
import batou_ext.nix


class Zeo(Component):
Expand All @@ -16,6 +17,41 @@ class Zeo(Component):
def configure(self):
self.provide("zeo:server", self.address)

self += batou_ext.nix.UserEnv(
"zope",
packages=[
"abiword",
"aespipe",
"cron",
"cyrus_sasl",
"enchant",
"gcc",
"gettext",
"ghostscript",
"gnumake",
"graphicsmagick",
"html-tidy",
"libedit",
"libffi",
"libjpeg",
"libxslt",
"libyaml",
"pdf2svg",
"pdftk",
"perl",
"perlPackages.LWPProtocolHttps",
"perlPackages.LWPUserAgent",
"perlPackages.MonitoringPlugin",
"poppler_utils",
"tmpwatch",
"wkhtmltopdf",
"wv",
"yarn",
"zlib",
],
channel="https://releases.nixos.org/nixos/19.03/nixos-19.03.173575.0e0ee084d6d/nixexprs.tar.xz",
)

self += Clone(
"https://github.com/syslabcom/recensio.buildout.git",
branch=self.branch,
Expand Down
169 changes: 106 additions & 63 deletions components/zope/component.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,102 +7,145 @@
from batou.lib.supervisor import Eventlistener
from batou.lib.supervisor import Program
from batou.utils import Address
import batou_ext.nix
import os.path


class DevelopAwareBuildout(Buildout):

def verify(self):
super(DevelopAwareBuildout, self).verify()
if os.path.exists('bin/develop'):
if os.path.exists("bin/develop"):
raise UpdateNeeded


class Zope(Component):
"""Deploys the zope instance with all dependencies.
"""
instance_address = Attribute(Address, '{{host.fqdn}}:8080')
instance1_address = Attribute(Address, '{{host.fqdn}}:8081')
instance2_address = Attribute(Address, '{{host.fqdn}}:8082')
instance3_address = Attribute(Address, '{{host.fqdn}}:8083')
instance4_address = Attribute(Address, '{{host.fqdn}}:8084')
instancebots_address = Attribute(Address, '{{host.fqdn}}:8089')
worker_address = Attribute(Address, '{{host.fqdn}}:8099')
solr_address = Attribute(Address, '{{host.fqdn}}:8983')
profile = 'base'
branch = 'master'

instance_address = Attribute(Address, "{{host.fqdn}}:8080")
instance1_address = Attribute(Address, "{{host.fqdn}}:8081")
instance2_address = Attribute(Address, "{{host.fqdn}}:8082")
instance3_address = Attribute(Address, "{{host.fqdn}}:8083")
instance4_address = Attribute(Address, "{{host.fqdn}}:8084")
instancebots_address = Attribute(Address, "{{host.fqdn}}:8089")
worker_address = Attribute(Address, "{{host.fqdn}}:8099")
solr_address = Attribute(Address, "{{host.fqdn}}:8983")
profile = "base"
branch = "master"
adminpw = None
portals = Attribute('literal', [])
sentry_dsn = Attribute(str, '')
manage_buildout_clone = Attribute('literal', True)
eggserver = ''
portals = Attribute("literal", [])
sentry_dsn = Attribute(str, "")
manage_buildout_clone = Attribute("literal", True)
eggserver = ""

features = ('instance', 'worker', 'instancebots', )
features = ("instance", "worker", "instancebots")
numbered_instances = Attribute(int, 0)

def configure(self):
# self.provide('zope:http', self.instance_address)
self.zeo = self.require_one('zeo:server')
self.solr = self.require_one('solr:server')
self.zeo = self.require_one("zeo:server")
self.solr = self.require_one("solr:server")

self.extra_parts = []
if 'instance' in self.features:
if "instance" in self.features:
for num in range(1, self.numbered_instances + 1):
self.extra_parts.append('instance{0}'.format(num))
if 'instancebots' in self.features:
self.extra_parts.append('instancebots')
if 'worker' in self.features:
self.extra_parts.append('worker')
if 'solr' in self.features:
self.extra_parts.extend(['solr-download', 'solr-instance'])

self += Directory('downloads')
buildout = DevelopAwareBuildout(python='2.7',
setuptools='38.5.1',
version='2.11.4',
additional_config=[])
self.extra_parts.append("instance{0}".format(num))
if "instancebots" in self.features:
self.extra_parts.append("instancebots")
if "worker" in self.features:
self.extra_parts.append("worker")
if "solr" in self.features:
self.extra_parts.extend(["solr-download", "solr-instance"])

self += batou_ext.nix.UserEnv(
"zope",
packages=[
"abiword",
"aespipe",
"cron",
"cyrus_sasl",
"enchant",
"gcc",
"gettext",
"ghostscript",
"gnumake",
"graphicsmagick",
"html-tidy",
"libedit",
"libffi",
"libjpeg",
"libxslt",
"libyaml",
"pdf2svg",
"pdftk",
"perl",
"perlPackages.LWPProtocolHttps",
"perlPackages.LWPUserAgent",
"perlPackages.MonitoringPlugin",
"poppler_utils",
"tmpwatch",
"wkhtmltopdf",
"wv",
"yarn",
"zlib",
],
channel="https://releases.nixos.org/nixos/19.03/nixos-19.03.173575.0e0ee084d6d/nixexprs.tar.xz",
)

self += Directory("downloads")
buildout = DevelopAwareBuildout(
python="2.7", setuptools="38.5.1", version="2.11.4", additional_config=[]
)
self += buildout
buildout += Clone('https://github.com/syslabcom/recensio.buildout.git',
branch=self.branch,
vcs_update=self.manage_buildout_clone)
buildout += Clone(
"https://github.com/syslabcom/recensio.buildout.git",
branch=self.branch,
vcs_update=self.manage_buildout_clone,
)

if 'instance' in self.features:
if "instance" in self.features:
for num in range(1, self.numbered_instances + 1):
instance_id = 'instance{0}'.format(num)
instance_id = "instance{0}".format(num)
self += Program(
instance_id,
options=dict(startsecs=20, stopsignal='INT', stopwaitsecs=5),
command=self.map('bin/{0} console'.format(instance_id)))
options=dict(startsecs=20, stopsignal="INT", stopwaitsecs=5),
command=self.map("bin/{0} console".format(instance_id)),
)

self += Eventlistener(
'memmon{0}'.format(num),
command='bin/memmon',
args='-p instance{0}=2GB -m [email protected]'.format(num))
"memmon{0}".format(num),
command="bin/memmon",
args="-p instance{0}=2GB -m [email protected]".format(num),
)

if 'instancebots' in self.features:
if "instancebots" in self.features:
self += Program(
'instancebots',
options=dict(startsecs=20, stopsignal='INT', stopwaitsecs=5),
command=self.map('bin/instancebots console'))
"instancebots",
options=dict(startsecs=20, stopsignal="INT", stopwaitsecs=5),
command=self.map("bin/instancebots console"),
)

self += Eventlistener(
'memmonbots',
command='bin/memmon',
args='-p instancebots=2GB')
"memmonbots", command="bin/memmon", args="-p instancebots=2GB"
)

if 'worker' in self.features:
if "worker" in self.features:
self += Program(
'worker',
options=dict(startsecs=20, stopsignal='INT', stopwaitsecs=5),
command=self.map('bin/worker console'))
"worker",
options=dict(startsecs=20, stopsignal="INT", stopwaitsecs=5),
command=self.map("bin/worker console"),
)

self += Eventlistener(
'memmonworker',
command='bin/memmon',
args='-p worker=2GB -m [email protected]')

if 'solr' in self.features:
self += Program('solr3',
command='/usr/bin/env java',
args='-Xms512m -Xmx2048m -jar start.jar',
directory=self.map('parts/solr-instance'))
"memmonworker",
command="bin/memmon",
args="-p worker=2GB -m [email protected]",
)

if "solr" in self.features:
self += Program(
"solr3",
command="/usr/bin/env java",
args="-Xms512m -Xmx2048m -jar start.jar",
directory=self.map("parts/solr-instance"),
)
Loading