Skip to content

Commit

Permalink
Add NUMA awareness to Lago.
Browse files Browse the repository at this point in the history 
Logic:

If there's >1 vCPU:
2 vCPUs - 2 NUMA nodes, each has 1 vCPU
4 vCPUS - 2 NUMA nodes, each has 1 vCPU

Otherwise:

If num of vCPUs (N) divisible by 4:
N/4 NUMA nodes, each has 4 vCPUs
If num of vCPUs divisible by 2:
N/2 NUMA nodes, each has 2 vCPUs

Otherwise,
N NUMA nodes, each has 1 vCPU.

Tested with 1 - 6 vCPUs.
  • Loading branch information
Yaniv Kaul committed Feb 26, 2018
1 parent 103d944 commit f31f792
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 31 deletions.
88 changes: 79 additions & 9 deletions lago/providers/libvirt/cpu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ def __init__(self, spec, host_cpu):
if spec.get('vcpu'):
self.vcpu_set = True
self.vcpu_num = spec['vcpu']
self.cpu = spec.get('cpu_custom')
self.cpu_custom = spec.get('cpu_custom')
self.cpu_model = spec.get('cpu_model')
self.host_cpu = host_cpu
self.memory = spec.get('memory')
self.validate()

self._cpu_xml = self.generate_cpu_xml()
Expand All @@ -61,10 +62,10 @@ def validate(self):
Validate CPU-related VM spec are compatible
Raises:
:exc:`~LagoInitException`: if both 'cpu_model' and 'cpu' are
:exc:`~LagoInitException`: if both 'cpu_model' and 'cpu_custom' are
defined.
"""
if self.cpu is not None and self.cpu_model:
if self.cpu_custom is not None and self.cpu_model:
raise LagoInitException(
'Defining both cpu_model and cpu_custom is '
'not supported.'
Expand All @@ -82,7 +83,7 @@ def vcpu_xml(self):
def model(self):
if self.cpu_model:
return self.cpu_model
elif not self.cpu:
elif not self.cpu_custom:
return self.host_cpu.xpath('model')[0].text
else:
return self._cpu_xml.xpath('model')[0].text
Expand All @@ -91,7 +92,7 @@ def model(self):
def vendor(self):
if self.cpu_model:
return LibvirtCPU.get_cpu_vendor(self.cpu_model)
elif not self.cpu:
elif not self.cpu_custom:
return self.host_cpu.xpath('vendor')[0].text
else:
return LibvirtCPU.get_cpu_vendor(self.model)
Expand All @@ -103,7 +104,7 @@ def generate_cpu_xml(self):
Returns:
lxml.etree.Element: cpu node
"""
if self.cpu:
if self.cpu_custom:
return self.generate_custom(
cpu=self.cpu,
vcpu_num=self.vcpu_num,
Expand Down Expand Up @@ -134,14 +135,16 @@ def generate_host_passthrough(self, vcpu_num):
Generate host-passthrough XML cpu node
Args:
vcpu_num(int): number of virtual CPUs
vcpu_num(str): number of virtual CPUs
Returns:
lxml.etree.Element: CPU XML node
"""

cpu = ET.Element('cpu', mode='host-passthrough')
cpu.append(self.generate_topology(vcpu_num))
if vcpu_num > 1:
cpu.append(self.generate_numa(vcpu_num))
return cpu

def generate_custom(self, cpu, vcpu_num, fill_topology):
Expand Down Expand Up @@ -229,7 +232,7 @@ def generate_topology(self, vcpu_num, cores=1, threads=1):
Generate CPU <topology> XML child
Args:
vcpu_num(int): number of virtual CPUs
vcpu_num(str): number of virtual CPUs
cores(int): number of cores
threads(int): number of threads
Expand All @@ -244,12 +247,79 @@ def generate_topology(self, vcpu_num, cores=1, threads=1):
threads=str(threads),
)

def generate_numa(self, vcpu_num):
"""
Generate guest CPU <numa> XML child
Configures 1, 2 or 4 vCPUs per cell.
Args:
vcpu_num(str): number of virtual CPUs
Returns:
lxml.etree.Element: numa XML element
"""

if int(vcpu_num) == 2:
# 2 vCPUs is a special case.
# We wish to have 2 cells,
# with 1 vCPU in each.
# This is also the common case.
total_cells = 2
cpus_per_cell = 1
elif int(vcpu_num) == 4:
# 4 vCPU is a special case.
# We wish to have 2 cells,
# with 2 vCPUs in each.
total_cells = 2
cpus_per_cell = 2
else:
cell_info = divmod(int(vcpu_num), 4)
if cell_info[1] == 0:
# 4 vCPUs in each cell
total_cells = cell_info[0]
cpus_per_cell = 4
elif cell_info[1] == 2:
# 2 vCPUs in each cell
total_cells = (cell_info[0] * 2) + 1
cpus_per_cell = 2
else:
# 1 vCPU per cell...
total_cells = int(vcpu_num)
cpus_per_cell = 1

numa = ET.Element('numa')
memory_per_cell = divmod(int(self.memory), total_cells)
LOGGER.debug(
'numa\n: cpus_per_cell: {0}, total_cells: {1}'.format(
cpus_per_cell, total_cells
)
)
for cell_id in xrange(0, total_cells):
first_cpu_in_cell = cell_id * cpus_per_cell
if cpus_per_cell == 1:
cpus_in_cell = str(first_cpu_in_cell)
else:
cpus_in_cell = '{0}-{1}'.format(
first_cpu_in_cell, first_cpu_in_cell + cpus_per_cell - 1
)
cell = ET.Element(
'cell',
id=str(cell_id),
cpus=cpus_in_cell,
memory=str(memory_per_cell[0]),
unit='MiB',
)
numa.append(cell)

LOGGER.debug('numa:\n{}'.format(ET.tostring(numa, pretty_print=True)))
return numa

def generate_vcpu(self, vcpu_num):
"""
Generate <vcpu> domain XML child
Args:
vcpu_num(int): number of virtual cpus
vcpu_num(str): number of virtual cpus
Returns:
lxml.etree.Element: vcpu XML element
Expand Down
95 changes: 73 additions & 22 deletions tests/unit/lago/providers/libvirt/test_cpu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def test_generate_topology(self):
} for tup in permutations(range(1, 4), 3)
]

empty_cpu = cpu.CPU(spec={}, host_cpu=None)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
for comb in combs:
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_topology(**comb)),
Expand All @@ -62,15 +62,54 @@ def test_generate_topology(self):
def test_generate_host_passthrough(self):
_xml = """
<cpu mode="host-passthrough">
<topology sockets="{0}" cores="1" threads="1"/>
<topology sockets="{0}" cores="1" threads="1"/>{1}
</cpu>
"""
empty_cpu = cpu.CPU(spec={}, host_cpu=None)
for vcpu_num in [1, 9, 11, 120]:
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_host_passthrough(vcpu_num)),
_xml.format(vcpu_num)
)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
vcpu_num = 1
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_host_passthrough(vcpu_num)),
_xml.format(vcpu_num, '')
)

_numa2 = """
<numa>
<cell cpus="0" id="0" memory="1023" unit="MiB"/>
<cell cpus="1" id="1" memory="1023" unit="MiB"/>
</numa>
"""
empty_cpu = cpu.CPU(spec={'memory': 2047}, host_cpu=None)
vcpu_num = 2
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_host_passthrough(vcpu_num)),
_xml.format(vcpu_num, _numa2)
)

_numa3 = """
<numa>
<cell cpus="0" id="0" memory="682" unit="MiB"/>
<cell cpus="1" id="1" memory="682" unit="MiB"/>
<cell cpus="2" id="2" memory="682" unit="MiB"/>
</numa>
"""
vcpu_num = 3
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_host_passthrough(vcpu_num)),
_xml.format(vcpu_num, _numa3)
)

_numa8 = """
<numa>
<cell cpus="0-3" id="0" memory="2048" unit="MiB"/>
<cell cpus="4-7" id="1" memory="2048" unit="MiB"/>
</numa>
"""
empty_cpu = cpu.CPU(spec={'memory': 4096}, host_cpu=None)
vcpu_num = 8
self.assertXmlEquivalentOutputs(
ET.tostring(empty_cpu.generate_host_passthrough(vcpu_num)),
_xml.format(vcpu_num, _numa8)
)

def test_generate_exact_intel_vmx_intel_vmx(self, vcpu=2, model='Penryn'):

Expand All @@ -93,11 +132,12 @@ def test_generate_exact_intel_vmx_intel_vmx(self, vcpu=2, model='Penryn'):
</cpu>
"""
)
empty_cpu = cpu.CPU(spec={}, host_cpu=None)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
self.assertXmlEquivalentOutputs(
ET.tostring(
empty_cpu.
generate_exact(model=model, vcpu_num=vcpu, host_cpu=host)
empty_cpu.generate_exact(
model=model, vcpu_num=vcpu, host_cpu=host
)
), _xml
)

Expand All @@ -120,11 +160,12 @@ def test_generate_exact_intel_novmx(self, vcpu=2, model='Penryn'):
</cpu>
"""
)
empty_cpu = cpu.CPU(spec={}, host_cpu=None)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
self.assertXmlEquivalentOutputs(
ET.tostring(
empty_cpu.
generate_exact(model=model, vcpu_num=vcpu, host_cpu=host)
empty_cpu.generate_exact(
model=model, vcpu_num=vcpu, host_cpu=host
)
), _xml
)

Expand All @@ -147,11 +188,12 @@ def test_generate_exact_vendor_mismatch(self, vcpu=2, model='Opteron_G2'):
</cpu>
"""
)
empty_cpu = cpu.CPU(spec={}, host_cpu=None)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
self.assertXmlEquivalentOutputs(
ET.tostring(
empty_cpu.
generate_exact(model=model, vcpu_num=vcpu, host_cpu=host)
empty_cpu.generate_exact(
model=model, vcpu_num=vcpu, host_cpu=host
)
), _xml
)

Expand All @@ -174,25 +216,34 @@ def test_generate_exact_unknown_vendor(self, vcpu=2, model='Westmere'):
</cpu>
"""
)
empty_cpu = cpu.CPU(spec={}, host_cpu=None)
empty_cpu = cpu.CPU(spec={'memory': 2048}, host_cpu=None)
self.assertXmlEquivalentOutputs(
ET.tostring(
empty_cpu.
generate_exact(model=model, vcpu_num=vcpu, host_cpu=host)
empty_cpu.generate_exact(
model=model, vcpu_num=vcpu, host_cpu=host
)
), _xml
)

def test_init_default(self):
spec = {}
spec = {'memory': 2048}
_xml = """
<cpu mode="host-passthrough">
<topology sockets="2" cores="1" threads="1"/>
<numa>
<cell cpus="0" id="0" memory="1024" unit="MiB"/>
<cell cpus="1" id="1" memory="1024" unit="MiB"/>
</numa>
</cpu>
"""
def_cpu = cpu.CPU(spec=spec, host_cpu=self.get_host_cpu())
self.assertXmlEquivalentOutputs(ET.tostring(def_cpu.cpu_xml), _xml)

def test_init_custom_and_model_not_allowed(self):
spec = {'cpu_custom': 'custom', 'cpu_model': 'DummyModel'}
spec = {
'cpu_custom': 'custom',
'cpu_model': 'DummyModel',
'memory': 2048
}
with pytest.raises(LagoInitException):
cpu.CPU(spec=spec, host_cpu=self.get_host_cpu())

0 comments on commit f31f792

Please sign in to comment.