diff --git a/tasks/config.yml b/tasks/config.yml
index 3dffca8..88506c5 100644
--- a/tasks/config.yml
+++ b/tasks/config.yml
@@ -6,6 +6,7 @@
     owner: root
     group: root
     mode: 0644
+  register: chrony_config
   notify: Restart chronyd
 
 - name: Ensure chronyd is running
@@ -13,3 +14,16 @@
     name: "{{ chrony_service_name }}"
     state: "{{ chrony_service_state|default('started') }}"
   when: chrony_manage_service|bool
+
+# ansible sometimes ignores handlers when invoked elsewhere. Since we will
+# likely want to restart chrony after a configuration update, let's do it here
+# always. The handler will still trigger for upgrades/updates later as well.
+# This won't have any effect if chrony_manage_service is set to false.
+# See also https://github.com/ansible/ansible/issues/37512
+- name: Force chronyd restart
+  service:
+    name: "{{ chrony_service_name }}"
+    state: restarted
+  when:
+    - chrony_manage_service|bool
+    - chrony_config.changed