Add Google Cloud Engine metadata service

Add cloudbaseinit.metadata.services.gceservice.GCEService that
supports Google Cloud Engine.

Supported features for the metadata service:
  * instance id
  * hostname
  * userdata
  * ssh keys

Change-Id: I9e5e2cbcaa0953fc0c0ae8117e258713ac6443b7

cloudbaseinit/conf/factory.py

@@ -25,6 +25,7 @@ _OPT_PATHS = (
     'cloudbaseinit.conf.ovf.OvfOptions',
     'cloudbaseinit.conf.packet.PacketOptions',
     'cloudbaseinit.conf.vmwareguestinfo.VMwareGuestInfoConfigOptions',
+    'cloudbaseinit.conf.gce.GCEOptions',
 )
 
 

cloudbaseinit/conf/gce.py

@@ -0,0 +1,51 @@
+# Copyright 2020 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Config options available for the GCE metadata service."""
+
+from oslo_config import cfg
+
+from cloudbaseinit.conf import base as conf_base
+
+
+class GCEOptions(conf_base.Options):
+
+    """Config options available for the GCE metadata service."""
+
+    def __init__(self, config):
+        super(GCEOptions, self).__init__(config, group="gce")
+        self._options = [
+            cfg.StrOpt(
+                "metadata_base_url",
+                default="http://metadata.google.internal/computeMetadata/v1/",
+                help="The base URL where the service looks for metadata"),
+            cfg.BoolOpt(
+                "https_allow_insecure", default=False,
+                help="Whether to disable the validation of HTTPS "
+                     "certificates."),
+            cfg.StrOpt(
+                "https_ca_bundle", default=None,
+                help="The path to a CA_BUNDLE file or directory with "
+                     "certificates of trusted CAs."),
+        ]
+
+    def register(self):
+        """Register the current options to the global ConfigOpts object."""
+        group = cfg.OptGroup(self.group_name, title='GCE Options')
+        self._config.register_group(group)
+        self._config.register_opts(self._options, group=group)
+
+    def list(self):
+        """Return a list which contains all the available options."""
+        return self._options

cloudbaseinit/metadata/services/gceservice.py

@@ -0,0 +1,184 @@
+# Copyright 2020 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import base64
+import json
+
+from datetime import datetime
+from oslo_log import log as oslo_logging
+
+from cloudbaseinit import conf as cloudbaseinit_conf
+from cloudbaseinit.metadata.services import base
+
+CONF = cloudbaseinit_conf.CONF
+LOG = oslo_logging.getLogger(__name__)
+
+GCE_METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
+MD_INSTANCE = "instance"
+MD_INSTANCE_ATTR = "%s/attributes" % MD_INSTANCE
+MD_PROJECT_ATTR = "project/attributes"
+
+
+class GCEService(base.BaseHTTPMetadataService):
+
+    def __init__(self):
+        super(GCEService, self).__init__(
+            base_url=CONF.gce.metadata_base_url,
+            https_allow_insecure=CONF.gce.https_allow_insecure,
+            https_ca_bundle=CONF.gce.https_ca_bundle)
+        self._enable_retry = True
+
+    def _http_request(self, url, data=None, headers=None, method=None):
+        headers = headers or {}
+        headers.update(GCE_METADATA_HEADERS)
+
+        return super(GCEService, self)._http_request(url, data,
+                                                     headers, method)
+
+    def load(self):
+        super(GCEService, self).load()
+
+        try:
+            self.get_host_name()
+            return True
+        except base.NotExistingMetadataException:
+            LOG.debug("Metadata not found at URL '%s'",
+                      CONF.gce.metadata_base_url)
+
+    def get_host_name(self):
+        return self._get_cache_data('%s/name' % MD_INSTANCE, decode=True)
+
+    def get_instance_id(self):
+        return self._get_cache_data('%s/id' % MD_INSTANCE, decode=True)
+
+    def get_user_data(self):
+        user_data = self._get_cache_data('%s/user-data' % MD_INSTANCE_ATTR)
+        try:
+            encoding = self._get_cache_data(
+                '%s/user-data-encoding' % MD_INSTANCE_ATTR,
+                decode=True)
+            if encoding:
+                if encoding == 'base64':
+                    user_data = base64.b64decode(user_data)
+                else:
+                    LOG.warning("Encoding '%s' not supported. "
+                                "Falling back to plaintext", encoding)
+        except base.NotExistingMetadataException:
+            LOG.info('Userdata encoding could not be found in the metadata.')
+
+        return user_data
+
+    def _is_ssh_key_valid(self, expire_on):
+        if not expire_on:
+            return True
+        try:
+            time_format = '%Y-%m-%dT%H:%M:%S+0000'
+            expire_time = datetime.strptime(expire_on, time_format)
+            return datetime.utcnow() <= expire_time
+        except ValueError:
+            # Note(ader1990): Return True to be consistent with cloud-init
+            return True
+
+    def _parse_gce_ssh_key(self, raw_ssh_key):
+        # GCE public keys have a special format defined here:
+        # https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#sshkeyformat
+        INVALID_SSH_KEY_MSG = "Skipping invalid SSH key %s"
+        header_username = None
+        meta_username = None
+
+        if not raw_ssh_key:
+            return
+
+        ssh_key = raw_ssh_key.strip()
+
+        # Key is in the format: USERNAME:ssh-rsa ...
+        # Remove the username from the key
+        ssh_key_split = ssh_key.split(':', 1)
+        if len(ssh_key_split) != 2:
+            LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
+            return
+
+        header_username, ssh_key = ssh_key_split
+
+        key_parts = ssh_key.split(' ')
+        len_key_parts = len(key_parts)
+
+        if len_key_parts < 3:
+            # Key format not supported: USERNAME:ssh-rsa [KEY]
+            LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
+            return
+        elif len_key_parts == 3:
+            # Key format: USERNAME:ssh-rsa [KEY] [USERNAME]
+            meta_username = key_parts[2]
+        else:
+            # Key format: USERNAME:ssh-rsa [KEY] google-ssh [JSON_METADATA]
+            delimiter = 'google-ssh'
+            json_key_parts = ssh_key.split(delimiter)
+            if (len(json_key_parts) == 2 and json_key_parts[1]):
+                ssh_key_metadata = json.loads(json_key_parts[1].strip())
+                meta_username = ssh_key_metadata['userName']
+                if not self._is_ssh_key_valid(ssh_key_metadata['expireOn']):
+                    LOG.warning("Skipping expired key: %s", ssh_key)
+                    return
+                ssh_key = '%s %s' % (json_key_parts[0].strip(), meta_username)
+            else:
+                LOG.warning(INVALID_SSH_KEY_MSG, ssh_key)
+                return
+
+        if not (header_username == meta_username == CONF.username):
+            LOG.warning("Skipping key due to non matching username: %s",
+                        ssh_key)
+            return
+
+        return ssh_key
+
+    def _get_ssh_keys(self, locations):
+        ssh_keys = []
+        for location in locations:
+            try:
+                raw_ssh_keys = self._get_cache_data(location, decode=True)
+                ssh_keys += raw_ssh_keys.strip().splitlines()
+            except base.NotExistingMetadataException:
+                LOG.warning("SSH keys not found at location %s", location)
+
+        return ssh_keys
+
+    def get_public_keys(self):
+        ssh_keys = []
+        raw_ssh_keys = []
+        block_project_keys = False
+        key_locations = ["%s/ssh-keys" % MD_INSTANCE_ATTR]
+
+        # Use GCE latest metadata, where the SSH keys are found
+        # only at hyphenated locations
+        try:
+            block_key = "%s/block-project-ssh-keys" % MD_INSTANCE_ATTR
+            if self._get_cache_data(block_key, decode=True) == 'true':
+                block_project_keys = True
+        except base.NotExistingMetadataException:
+                LOG.debug('block-project-ssh-keys not present')
+
+        if not block_project_keys:
+            key_locations += [
+                "%s/ssh-keys" % MD_PROJECT_ATTR
+            ]
+
+        raw_ssh_keys += self._get_ssh_keys(key_locations)
+
+        for raw_ssh_key in raw_ssh_keys:
+            ssh_key = self._parse_gce_ssh_key(raw_ssh_key)
+            if ssh_key:
+                ssh_keys.append(ssh_key)
+
+        return ssh_keys

cloudbaseinit/tests/metadata/services/test_gceservice.py

@@ -0,0 +1,172 @@
+# Copyright 2020 Cloudbase Solutions Srl
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import ddt
+import importlib
+import unittest
+
+try:
+    import unittest.mock as mock
+except ImportError:
+    import mock
+
+from cloudbaseinit import conf as cloudbaseinit_conf
+from cloudbaseinit.tests import testutils
+
+
+CONF = cloudbaseinit_conf.CONF
+BASE_MODULE_PATH = ("cloudbaseinit.metadata.services.base."
+                    "BaseHTTPMetadataService")
+MODULE_PATH = "cloudbaseinit.metadata.services.gceservice"
+
+
+@ddt.ddt
+class GCEServiceTest(unittest.TestCase):
+
+    def setUp(self):
+        self._win32com_mock = mock.MagicMock()
+        self._ctypes_mock = mock.MagicMock()
+        self._ctypes_util_mock = mock.MagicMock()
+        self._win32com_client_mock = mock.MagicMock()
+        self._pywintypes_mock = mock.MagicMock()
+        self._module_patcher = mock.patch.dict(
+            'sys.modules',
+            {'win32com': self._win32com_mock,
+             'ctypes': self._ctypes_mock,
+             'ctypes.util': self._ctypes_util_mock,
+             'win32com.client': self._win32com_client_mock,
+             'pywintypes': self._pywintypes_mock})
+        self._module_patcher.start()
+        self.addCleanup(self._module_patcher.stop)
+
+        self._module = importlib.import_module(MODULE_PATH)
+        self._service = self._module.GCEService()
+        self.snatcher = testutils.LogSnatcher(MODULE_PATH)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    def test_get_host_name(self, mock_get_cache_data):
+        response = self._service.get_host_name()
+        mock_get_cache_data.assert_called_once_with(
+            'instance/name', decode=True)
+        self.assertEqual(mock_get_cache_data.return_value,
+                         response)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    def test_get_instance_id(self, mock_get_cache_data):
+        response = self._service.get_instance_id()
+        mock_get_cache_data.assert_called_once_with(
+            'instance/id', decode=True)
+        self.assertEqual(mock_get_cache_data.return_value,
+                         response)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    def test_get_user_data(self, mock_get_cache_data):
+        response = self._service.get_user_data()
+        userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR
+        userdata_enc_key = (
+            "%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR)
+        mock_calls = [mock.call(userdata_key),
+                      mock.call(userdata_enc_key, decode=True)]
+        mock_get_cache_data.assert_has_calls(mock_calls)
+        self.assertEqual(mock_get_cache_data.return_value,
+                         response)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    def test_get_user_data_b64(self, mock_get_cache_data):
+        user_data = b'fake userdata'
+        user_data_b64 = 'ZmFrZSB1c2VyZGF0YQ=='
+        userdata_key = "%s/user-data" % self._module.MD_INSTANCE_ATTR
+        userdata_enc_key = (
+            "%s/user-data-encoding" % self._module.MD_INSTANCE_ATTR)
+
+        def _get_cache_data_side_effect(*args, **kwargs):
+            if args[0] == ("%s/user-data" % self._module.MD_INSTANCE_ATTR):
+                return user_data_b64
+            return 'base64'
+        mock_get_cache_data.side_effect = _get_cache_data_side_effect
+
+        response = self._service.get_user_data()
+
+        mock_calls = [mock.call(userdata_key),
+                      mock.call(userdata_enc_key, decode=True)]
+        mock_get_cache_data.assert_has_calls(mock_calls)
+        self.assertEqual(response, user_data)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    @mock.patch(MODULE_PATH + ".GCEService._get_ssh_keys")
+    @mock.patch(MODULE_PATH + ".GCEService._parse_gce_ssh_key")
+    def _test_get_public_keys_block_project(self, mock_parse_keys,
+                                            mock_get_ssh_keys,
+                                            mock_cache_data,
+                                            cache_data_result=False):
+        expected_response = []
+
+        if cache_data_result:
+            second_call_get_ssh = [
+                '%s/ssh-keys' % self._module.MD_INSTANCE_ATTR,
+                '%s/ssh-keys' % self._module.MD_PROJECT_ATTR]
+            mock_cache_data.return_value = 'false'
+        else:
+            second_call_get_ssh = [
+                '%s/ssh-keys' % self._module.MD_INSTANCE_ATTR]
+            mock_cache_data.return_value = 'true'
+        mock_get_ssh_keys.return_value = []
+        response = self._service.get_public_keys()
+        mock_calls = [mock.call(second_call_get_ssh)]
+        mock_get_ssh_keys.assert_has_calls(mock_calls)
+
+        self.assertEqual(mock_parse_keys.call_count, 0)
+        self.assertEqual(response, expected_response)
+
+    def test_get_public_keys_block_project_check(self):
+        self._test_get_public_keys_block_project(cache_data_result=False)
+
+    def test_get_public_keys_block_project(self):
+        self._test_get_public_keys_block_project(cache_data_result=True)
+
+    @mock.patch(MODULE_PATH + ".GCEService._get_cache_data")
+    def test__get_ssh_keys(self, mock_get_cache_data):
+        fake_key = 'fake key'
+        expected_response = [fake_key] * 3
+        key_locations = ['location'] * 3
+        mock_get_cache_data.return_value = fake_key
+        response = self._service._get_ssh_keys(key_locations)
+        self.assertEqual(response, expected_response)
+
+    @ddt.data((None, True),
+              ('not a date', True),
+              ('2018-12-04T20:12:00+0000', False))
+    @ddt.unpack
+    def test__is_ssh_key_valid(self, expire_on, expected_response):
+        response = self._service._is_ssh_key_valid(expire_on)
+        self.assertEqual(response, expected_response)
+
+    @ddt.data((None, None),
+              ('ssh invalid', None),
+              ('notadmin:ssh key notadmin', None),
+              ('Admin:ssh key Admin', 'ssh key Admin'),
+              ('ssh key google-ssh', None),
+              ('Admin:s k google-ssh {"userName":"Admin",'
+               '"expireOn":"1018-12-04T20:12:00+0000"}',
+               None),
+              ('Admin:s k google-ssh {"userName":"b",'
+               '"expireOn":"3018-12-04T20:12:00+0000"}',
+               None),
+              ('Admin:s k google-ssh {"userName":"Admin",'
+               '"expireOn":"3018-12-04T20:12:00+0000"}',
+               's k Admin'))
+    @ddt.unpack
+    def test__parse_gce_ssh_key(self, raw_ssh_key, expected_response):
+        response = self._service._parse_gce_ssh_key(raw_ssh_key)
+        self.assertEqual(response, expected_response)

doc/source/services.rst

@@ -433,3 +433,37 @@ Capabilities:
 Config options for `vmwareguestinfo` section:
 
     * vmware_rpctool_path (string: "%ProgramFiles%/VMware/VMware Tools/rpctool.exe")
+
+
+Google Compute Engine Service
+-----------------------------
+
+.. class:: cloudbaseinit.metadata.services.gceservice.GCEService
+
+`GCE <https://cloud.google.com/compute/>`_ metadata service provides
+the metadata for instances running on Google Compute Engine.
+
+GCE metadata is offered via an internal HTTP metadata endpoint, reachable at the magic URL
+`http://metadata.google.internal/computeMetadata/v1/`. More information can be found in the GCE
+metadata `documents <https://cloud.google.com/compute/docs/storing-retrieving-metadata#querying>`_.
+
+To provide userdata to be executed by the instance (in cloud-config format, for example), use the
+user-data and user-data-encoding instance metadata keys.
+
+Capabilities:
+
+    * instance id
+    * hostname
+    * public keys
+    * user data
+
+Config options for `gce` section:
+
+    * metadata_base_url (string: http://metadata.google.internal/computeMetadata/v1/")
+    * https_allow_insecure (bool: False)
+    * https_ca_bundle (string: None)
+
+Config options for `default` section:
+
+    * retry_count (integer: 5)
+    * retry_count_interval (integer: 4)