Parameters exploration

This tutorial illustrates how parameters can be explored. The cornerstone here is the excellent Execo ParamSweeper. For the illustration purpose we consider the following question:

  • How the network performance is impacted by concurrent network flows and different network conditions ?

To answer this question we’ll simply measure the network throughput while varying the number of virtual machines pairs and network characteristics between them. For this purpose, we reserve 2 physical machines: one for the source (client) virtual machines and the other for the destination (server) virtual machines. Every single source virtual machine is then paired with its destination virtual machine. Different network delays will be applied between the sources and the destination.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
from enoslib.api import play_on, wait_ssh, ensure_python3, discover_networks
from enoslib.infra.enos_vmong5k.provider import VMonG5k
from enoslib.infra.enos_vmong5k.configuration import Configuration
from enoslib.service import Netem


from execo_engine import ParamSweeper, sweep

import logging
from operator import itemgetter
import os
from pathlib import Path
import traceback
from typing import Dict


logging.basicConfig(level=logging.DEBUG)

CLUSTER = "paranoia"


def bench(parameter: Dict) -> None:
    """Launch a benchmark.

    1. Start the required ressources.
    2. Prepare the virtual machines
    3. Benchmark the network
    4. Backup
    """
    nb_vms = parameter["nb_vms"]
    conf = (
        Configuration
        .from_settings(force_deploy=True)
        .add_machine(roles=["server"],
                    cluster=CLUSTER,
                    number=nb_vms,
                    flavour="tiny")
        .add_machine(roles=["client"],
                    cluster=CLUSTER,
                    number=nb_vms,
                    flavour="tiny")
        .finalize()
    )

    provider = VMonG5k(conf)

    roles, networks = provider.init()
    roles = discover_networks(roles, networks)

    servers = roles["server"]
    clients = roles["client"]

    for s, c in zip(servers, clients):
        c.extra.update(target=s.address)

    ensure_python3(roles=roles)

    with play_on(roles=roles) as p:
        p.apt_repository(repo="deb http://deb.debian.org/debian stretch main contrib non-free",
                        state="present")
        p.apt(
            name=[
                "flent",
                "netperf",
                "python3-setuptools",
                "python3-matplotlib",
                "tmux"
            ],
            state="present")

    with play_on(pattern_hosts="server", roles=roles) as p:
        p.shell("tmux new-session -d 'exec netperf'")

    delay = parameter["delay"]
    if delay is not None:
        tc = dict(default_delay=delay, default_rate="10gbit", enabled=True)
        netem = Netem(tc, roles=roles)
        netem.deploy()
    output = f"tcp_upload_{nb_vms}_{delay}"
    with play_on(pattern_hosts="client", roles=roles) as p:
        p.shell(
            "flent tcp_upload -p totals "
            + "-l 60 "
            + "-H {{ target }} "
            + f"-t '{output}' "
            + f"-o {output}.png",
            display_name=f"Benchmarkings with {output}")
        p.fetch(src=f"{output}.png", dest=f"result_{output}")


parameters = dict(
    nb_vms=[1, 2, 4, 8, 16],
    delay=[None, "0ms", "10ms", "50ms"]
)


sweeps = sweep(parameters)
sweeper = ParamSweeper(
    persistence_dir=str(Path("sweeps")), sweeps=sweeps, save_sweeps=True
)

parameter = sweeper.get_next()
while parameter:
    try:
        print(parameter)
        bench(parameter)
        sweeper.done(parameter)
    except Exception as e:
        traceback.print_exc()
        sweeper.skip(parameter)
    finally:
        parameter = sweeper.get_next()