diff --git a/jenkins_jobs/config.py b/jenkins_jobs/config.py
index 6f146c190..2252392bd 100644
--- a/jenkins_jobs/config.py
+++ b/jenkins_jobs/config.py
@@ -141,6 +141,9 @@ class JJBConfig(object):
         self._setup()
         self._handle_deprecated_hipchat_config()
 
+        if config_fp is not None:
+            config_fp.close()
+
     def _init_defaults(self):
         """ Initialize default configuration values using DEFAULT_CONF
         """
diff --git a/tests/base.py b/tests/base.py
index 514b83c42..b239f64e3 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -137,7 +137,8 @@ class BaseTestCase(testtools.TestCase):
         # Read XML content, assuming it is unicode encoded
         xml_content = ""
         for f in sorted(self.out_filenames):
-            xml_content += u"%s" % io.open(f, 'r', encoding='utf-8').read()
+            with io.open(f, 'r', encoding='utf-8') as xml_file:
+                xml_content += u"%s" % xml_file.read()
         return xml_content
 
     def _read_yaml_content(self, filename):