From 049547fec5faedf6127cd7bf3c6e72f5a2fc16ab Mon Sep 17 00:00:00 2001 From: Jim Somerville Date: Fri, 26 Apr 2019 17:41:04 -0300 Subject: [PATCH] STX: migration thread affinity and priority qmp This includes de-blacklisting the scheduler and affinity setting syscalls. Signed-off-by: Jim Somerville [ Update hmp-commands struc ] Signed-off-by: Rafael Falcao [Rebased original changes to the qemu 7.2 version] Signed-off-by: david.liu --- hmp-commands.hx | 30 ++++++++++++++++++++++++ include/monitor/hmp.h | 2 ++ migration/migration.c | 52 ++++++++++++++++++++++++++++++++++++++++++ monitor/hmp-cmds.c | 25 ++++++++++++++++++++ qapi/misc.json | 30 ++++++++++++++++++++++++ softmmu/qemu-seccomp.c | 5 ---- softmmu/trace-events | 1 + 7 files changed, 140 insertions(+), 5 deletions(-) diff --git a/hmp-commands.hx b/hmp-commands.hx index 673e39a6..a00f3157 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -39,6 +39,36 @@ SRST completes. ERST + { + .name = "migrate-set-thread-cpumask", + .args_type = "value:o", + .params = "value", + .help = "Set CPU mask for the migration thread." + "Defaults to CPU 0 if no mask is specified", + .cmd = hmp_migrate_set_thread_cpumask, + }, + +SRST +``migrate_set_thread_cpumask`` *value* + Set CPU mask for the migration thread +ERST + + { + .name = "migrate-set-thread-priority", + .args_type = "value:o", + .params = "value", + .help = "Set real time priority for the the migration thread." + "Defaults to no change migration thread priority if not" + "specified or out of range. Range [1-99].Scheduling" + "policy will always be- SCHED_FIFO", + .cmd = hmp_migrate_set_thread_priority, + }, + +SRST +``migrate_set_thread_priority`` *value* + Set real time priority for the the migration thread +ERST + { .name = "quit|q", .args_type = "", diff --git a/include/monitor/hmp.h b/include/monitor/hmp.h index dfbc0c9a..394e90db 100644 --- a/include/monitor/hmp.h +++ b/include/monitor/hmp.h @@ -70,6 +70,8 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict); void hmp_client_migrate_info(Monitor *mon, const QDict *qdict); void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict); void hmp_x_colo_lost_heartbeat(Monitor *mon, const QDict *qdict); +void hmp_migrate_set_thread_cpumask(Monitor *mon, const QDict *qdict); +void hmp_migrate_set_thread_priority(Monitor *mon, const QDict *qdict); void hmp_set_password(Monitor *mon, const QDict *qdict); void hmp_expire_password(Monitor *mon, const QDict *qdict); void hmp_change(Monitor *mon, const QDict *qdict); diff --git a/migration/migration.c b/migration/migration.c index f485eea5..45f7b5d5 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -38,6 +38,7 @@ #include "qapi/qapi-visit-migration.h" #include "qapi/qapi-visit-sockets.h" #include "qapi/qapi-commands-migration.h" +#include "qapi/qapi-commands-misc.h" #include "qapi/qapi-events-migration.h" #include "qapi/qmp/qerror.h" #include "qapi/qmp/qnull.h" @@ -61,6 +62,15 @@ #include "sysemu/cpus.h" #include "yank_functions.h" #include "sysemu/qtest.h" +#include + +/* #define DEBUG */ +#ifdef DEBUG +#define DPRINTF(fmt, ...) \ + printf(fmt, ## __VA_ARGS__) +#else +#define DPRINTF(fmt, ...) +#endif #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */ @@ -116,6 +126,11 @@ #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS 5 #define DEFAULT_MIGRATE_ANNOUNCE_STEP 100 +/* variables for pinning the migration thread to a CPU and assigning the +* realtime priority to it */ +static uint64_t migrate_thread_cpumask=0; +static uint64_t migrate_thread_priority=0; + static NotifierList migration_state_notifiers = NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); @@ -2180,6 +2195,30 @@ void migrate_init(MigrationState *s) s->threshold_size = 0; } +void qmp_migrate_set_thread_cpumask(int64_t value, Error **errp) +{ + /* Check for truncation */ + if (value != (size_t)value) { + error_setg(errp, "Migration thread CPU Mask exceeding address space"); + return; + } + /*resize the value */ + value >>= 20; /*Magic */ + migrate_thread_cpumask = value; +} + +void qmp_migrate_set_thread_priority(int64_t value, Error **errp) +{ + /* Check for truncation */ + if (value != (size_t)value) { + error_setg(errp, "Migration thread Priority exceeding address space"); + return; + } + /*resize the value */ + value >>= 20; + migrate_thread_priority = value; +} + int migrate_add_blocker_internal(Error *reason, Error **errp) { /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */ @@ -3988,6 +4027,19 @@ static void *migration_thread(void *opaque) qemu_savevm_send_postcopy_advise(s->to_dst_file); } + /* Bind Migration thread to the processor specified by the user */ + if (sched_setaffinity(0, sizeof(migrate_thread_cpumask), (cpu_set_t *)&migrate_thread_cpumask) <0) { + DPRINTF("Error setting user input affinity. Switching to default.\n"); + } + + /* Change the realtime priority of the migration thread specified by the user */ + struct sched_param schedp; + memset(&schedp, 0, sizeof(schedp)); + schedp.sched_priority = migrate_thread_priority; + if (sched_setscheduler(0, SCHED_FIFO, &schedp) < 0) { + DPRINTF("Error setting user input priority. Switching to default.\n"); + } + if (migrate_colo_enabled()) { /* Notify migration destination that we enable COLO */ qemu_savevm_send_colo_enable(s->to_dst_file); diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 01b789a7..5395c64f 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -1444,6 +1444,31 @@ out: hmp_handle_error(mon, err); } +void hmp_migrate_set_thread_cpumask(Monitor *mon, const QDict *qdict) +{ + int64_t value = qdict_get_int(qdict, "value"); + Error *err = NULL; + + qmp_migrate_set_thread_cpumask(value, &err); + if (err) { + monitor_printf(mon, "%s\n", error_get_pretty(err)); + error_free(err); + return; + } +} + +void hmp_migrate_set_thread_priority(Monitor *mon, const QDict *qdict) +{ + int64_t value = qdict_get_int(qdict, "value"); + Error *err = NULL; + + qmp_migrate_set_thread_priority(value, &err); + if (err) { + monitor_printf(mon, "%s\n", error_get_pretty(err)); + error_free(err); + return; + } +} #ifdef CONFIG_VNC static void hmp_change_read_arg(void *opaque, const char *password, diff --git a/qapi/misc.json b/qapi/misc.json index 27ef5a2b..0efe14c5 100644 --- a/qapi/misc.json +++ b/qapi/misc.json @@ -246,6 +246,36 @@ 'returns': 'str', 'features': [ 'savevm-monitor-nodes' ] } +## +# @migrate-set-thread-cpumask: +# +# Set migration thread CPU mask. +# +# @value: CPU mask. +# +# Returns: nothing on success +# +# Notes: A value lesser than zero will be automatically round up to zero. +# +# Since: 0.14.0 +## +{ 'command': 'migrate-set-thread-cpumask', 'data': {'value': 'int'} } + +## +# @migrate-set-thread-priority: +# +# Set migration thread Real Timer priority mask. +# +# @value: Thread Priority. +# +# Returns: nothing on success +# +# Notes: A value lesser than zero will be automatically round up to zero. +# +# Since: 0.14.0 +## +{ 'command': 'migrate-set-thread-priority', 'data': {'value': 'int'} } + ## # @getfd: # diff --git a/softmmu/qemu-seccomp.c b/softmmu/qemu-seccomp.c index d66a2a12..2bfbcea8 100644 --- a/softmmu/qemu-seccomp.c +++ b/softmmu/qemu-seccomp.c @@ -258,11 +258,6 @@ static const struct QemuSeccompSyscall denylist[] = { 0, NULL, SCMP_ACT_ERRNO(EPERM) }, { SCMP_SYS(sched_setparam), QEMU_SECCOMP_SET_RESOURCECTL, 0, NULL, SCMP_ACT_ERRNO(EPERM) }, - { SCMP_SYS(sched_setscheduler), QEMU_SECCOMP_SET_RESOURCECTL, - ARRAY_SIZE(sched_setscheduler_arg), sched_setscheduler_arg, - SCMP_ACT_ERRNO(EPERM) }, - { SCMP_SYS(sched_setaffinity), QEMU_SECCOMP_SET_RESOURCECTL, - 0, NULL, SCMP_ACT_ERRNO(EPERM) }, }; static inline __attribute__((unused)) int diff --git a/softmmu/trace-events b/softmmu/trace-events index 22606dc2..ec100a33 100644 --- a/softmmu/trace-events +++ b/softmmu/trace-events @@ -31,6 +31,7 @@ runstate_set(int current_state, const char *current_state_str, int new_state, co system_wakeup_request(int reason) "reason=%d" qemu_system_shutdown_request(int reason) "reason=%d" qemu_system_powerdown_request(void) "" +migrate_thread(uint64_t migrate_thread_cpumask, uint64_t migrate_thread_priority) "migration Thread pinned to %" PRIu64 "with Priority %" PRIu64 #dirtylimit.c dirtylimit_state_initialize(int max_cpus) "dirtylimit state initialize: max cpus %d" -- 2.30.2