diff --git a/.travis.yml b/.travis.yml index e30ceae..dc0d6f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,9 @@ python: before_install: - "export NO_EVENTLET=1" install: - - pip install -r requirements.txt -r test-requirements.txt . --use-mirrors + - pip install tox --use-mirrors before_script: - - "flake8 --show-source --builtins=_ quark" + - mysql -e 'create database quark_functional_tests;' script: - - nosetests --with-coverage --cover-package=quark --cover-erase --cover-html --cover-html-dir=.cover-report --cover-min-percentage=90 + - tox sudo: false diff --git a/quark/db/api.py b/quark/db/api.py index 58c6e42..54f8617 100644 --- a/quark/db/api.py +++ b/quark/db/api.py @@ -29,6 +29,7 @@ from sqlalchemy import and_, asc, desc, orm, or_, not_ from sqlalchemy.orm import class_mapper from quark.db import models +from quark.db import sqlalchemy_adapter as quark_sa from quark import network_strategy from quark import protocols @@ -144,6 +145,7 @@ def _model_query(context, model, filters, fields=None): elif key == "reuse_after": reuse = (timeutils.utcnow() - datetime.timedelta(seconds=value)) + # NOTE(asadoughi): should this allow for deallocated_at = null? model_filters.append(model.deallocated_at <= reuse) elif key == "tenant_id": if model == models.IPAddress: @@ -314,9 +316,71 @@ def ip_address_find(context, lock_mode=False, **filters): model_filters.append( models.IPAddress.address_type == filters['address_type']) + if filters.get("transaction_id"): + model_filters.append( + models.IPAddress.transaction_id == filters['transaction_id']) + return query.filter(*model_filters) +@scoped +def ip_address_reallocate(context, update_kwargs, **filters): + LOG.debug("ip_address_reallocate %s", filters) + query = context.session.query(models.IPAddress) + model_filters = _model_query(context, models.IPAddress, filters) + query = query.filter(*model_filters) + row_count = quark_sa.update(query, update_kwargs, + update_args={"mysql_limit": 1}) + return row_count == 1 + + +def ip_address_reallocate_find(context, transaction_id): + address = ip_address_find(context, transaction_id=transaction_id, + scope=ONE) + if not address: + LOG.warn("Couldn't find IP address with transaction_id %s", + transaction_id) + return + + LOG.info("Potentially reallocatable IP found: " + "{0}".format(address["address_readable"])) + subnet = address.get('subnet') + if not subnet: + LOG.debug("No subnet associated with address") + return + if subnet["do_not_use"]: + LOG.debug("Subnet marked as do_not_use") + return + + addr = netaddr.IPAddress(int(address["address"])) + if address["subnet"]["ip_version"] == 4: + addr = addr.ipv4() + else: + addr = addr.ipv6() + + # TODO(amir): performance test replacing this with SQL in + # ip_address_reallocate's UPDATE statement + policy = models.IPPolicy.get_ip_policy_cidrs(subnet) + if policy is not None and addr in policy: + LOG.info("Deleting Address {0} due to policy " + "violation".format( + address["address_readable"])) + context.session.delete(address) + return + + # TODO(amir): performance test replacing this with SQL in + # ip_address_reallocate's UPDATE statement + cidr = netaddr.IPNetwork(address["subnet"]["cidr"]) + if addr not in cidr: + LOG.info("Address {0} isn't in the subnet " + "it claims to be in".format( + address["address_readable"])) + context.session.delete(address) + return + + return address + + @scoped def mac_address_find(context, lock_mode=False, **filters): query = context.session.query(models.MacAddress) @@ -760,3 +824,9 @@ def ip_policy_update(context, ip_policy, **ip_policy_dict): def ip_policy_delete(context, ip_policy): context.session.delete(ip_policy) + + +def transaction_create(context): + transaction = models.Transaction() + context.session.add(transaction) + return transaction diff --git a/quark/db/migration/alembic/versions/5632aa202d89_added_transactions_table.py b/quark/db/migration/alembic/versions/5632aa202d89_added_transactions_table.py new file mode 100644 index 0000000..1a4b993 --- /dev/null +++ b/quark/db/migration/alembic/versions/5632aa202d89_added_transactions_table.py @@ -0,0 +1,36 @@ +"""Added transactions table + +Revision ID: 5632aa202d89 +Revises: 3a47813ce501 +Create Date: 2015-03-18 14:54:09.061787 + +""" + +# revision identifiers, used by Alembic. +revision = '5632aa202d89' +down_revision = '4d3ed7925db3' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('quark_transactions', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB') + op.add_column(u'quark_ip_addresses', + sa.Column('transaction_id', sa.Integer(), nullable=True)) + op.create_foreign_key('fk_quark_ips_transaction_id', + 'quark_ip_addresses', + 'quark_transactions', + ['transaction_id'], + ['id']) + + +def downgrade(): + op.drop_constraint('fk_quark_ips_transaction_id', 'quark_ip_addresses', + type_='foreignkey') + op.drop_column(u'quark_ip_addresses', 'transaction_id') + op.drop_table('quark_transactions') diff --git a/quark/db/migration/alembic/versions/HEAD b/quark/db/migration/alembic/versions/HEAD index e3056e0..31df80e 100644 --- a/quark/db/migration/alembic/versions/HEAD +++ b/quark/db/migration/alembic/versions/HEAD @@ -1 +1 @@ -4d3ed7925db3 +5632aa202d89 diff --git a/quark/db/models.py b/quark/db/models.py index 2ebd783..3ea974d 100644 --- a/quark/db/models.py +++ b/quark/db/models.py @@ -163,6 +163,9 @@ class IPAddress(BASEV2, models.HasId): ip_types.SHARED, name="quark_ip_address_types")) associations = orm.relationship(PortIpAssociation, backref="ip_address") + transaction_id = sa.Column(sa.Integer(), + sa.ForeignKey("quark_transactions.id"), + nullable=True) def enabled_for_port(self, port): for assoc in self["associations"]: @@ -467,3 +470,8 @@ class Network(BASEV2, models.HasId): network_plugin = sa.Column(sa.String(36)) ipam_strategy = sa.Column(sa.String(255)) tenant_id = sa.Column(sa.String(255), index=True) + + +class Transaction(BASEV2): + __tablename__ = "quark_transactions" + id = sa.Column(sa.Integer, primary_key=True) diff --git a/quark/db/sqlalchemy_adapter.py b/quark/db/sqlalchemy_adapter.py new file mode 100644 index 0000000..a600615 --- /dev/null +++ b/quark/db/sqlalchemy_adapter.py @@ -0,0 +1,25 @@ +from sqlalchemy.orm.persistence import BulkUpdate +from sqlalchemy import sql + + +# NOTE(asadoughi): based on https://github.com/zzzeek/sqlalchemy/pull/164 +class BulkUpdateArgs(BulkUpdate): + def __init__(self, query, values, update_kwargs): + super(BulkUpdateArgs, self).__init__(query, values) + self.update_kwargs = update_kwargs + + def _do_exec(self): + update_stmt = sql.update(self.primary_table, + whereclause=self.context.whereclause, + values=self.values, + **self.update_kwargs) + self.result = self.query.session.execute( + update_stmt, params=self.query._params) + self.rowcount = self.result.rowcount + + +def update(query, values, update_args=None): + update_args = update_args or {} + update_op = BulkUpdateArgs(query, values, update_args) + update_op.exec_() + return update_op.rowcount diff --git a/quark/ipam.py b/quark/ipam.py index 0dc4387..65924db 100644 --- a/quark/ipam.py +++ b/quark/ipam.py @@ -352,104 +352,68 @@ class QuarkIpam(object): sub_ids = [] if subnets: sub_ids = subnets - else: - if segment_id: - subnets = db_api.subnet_find(elevated, - network_id=net_id, - segment_id=segment_id) - sub_ids = [s["id"] for s in subnets] - if not sub_ids: - LOG.info("No subnets matching segment_id {0} could be " - "found".format(segment_id)) - raise exceptions.IpAddressGenerationFailure( - net_id=net_id) + elif segment_id: + subnets = db_api.subnet_find(elevated, + network_id=net_id, + segment_id=segment_id) + sub_ids = [s["id"] for s in subnets] + if not sub_ids: + LOG.info("No subnets matching segment_id {0} could be " + "found".format(segment_id)) + raise exceptions.IpAddressGenerationFailure( + net_id=net_id) ip_kwargs = { - "network_id": net_id, "reuse_after": reuse_after, - "deallocated": True, "scope": db_api.ONE, - "ip_address": ip_address, "lock_mode": True, - "version": version, "order_by": "address"} - + "network_id": net_id, + "reuse_after": reuse_after, + "deallocated": True, + "ip_address": ip_address, + "version": version, + } if ip_address: del ip_kwargs["deallocated"] - if sub_ids: ip_kwargs["subnet_id"] = sub_ids + ipam_log = kwargs.get('ipam_log', None) - # We never want to take the chance of an infinite loop here. Instead, - # we'll clean up multiple bad IPs if we find them (assuming something - # is really wrong) for retry in xrange(CONF.QUARK.ip_address_retry_max): attempt = None if ipam_log: attempt = ipam_log.make_entry("attempt_to_reallocate_ip") LOG.info("Attempt {0} of {1}".format( retry + 1, CONF.QUARK.ip_address_retry_max)) - get_policy = models.IPPolicy.get_ip_policy_cidrs - try: with context.session.begin(): - # NOTE(mdietz): Before I removed the lazy=joined, this - # raised with an unknown column "address" - # error. - address = db_api.ip_address_find(elevated, **ip_kwargs) + transaction = db_api.transaction_create(context) + m = models.IPAddress + update_kwargs = { + m.transaction_id: transaction.id, + m.address_type: kwargs.get("address_type", ip_types.FIXED), + m.deallocated: False, + m.deallocated_at: None, + m.used_by_tenant_id: context.tenant_id, + m.allocated_at: timeutils.utcnow(), + } + result = db_api.ip_address_reallocate( + elevated, update_kwargs, **ip_kwargs) + if not result: + LOG.info("Couldn't update any reallocatable addresses " + "given the criteria") + if attempt: + attempt.failed() + break - if address: - # NOTE(mdietz): We should always be in the CIDR but we - # also said that before :-/ - LOG.info("Potentially reallocatable IP found: " - "{0}".format(address["address_readable"])) - subnet = address.get('subnet') - if subnet: - if subnet["do_not_use"]: - continue + updated_address = db_api.ip_address_reallocate_find( + elevated, transaction.id) + if not updated_address: + if attempt: + attempt.failed() + continue - policy = get_policy(subnet) - - cidr = netaddr.IPNetwork(address["subnet"]["cidr"]) - addr = netaddr.IPAddress(int(address["address"])) - if address["subnet"]["ip_version"] == 4: - addr = addr.ipv4() - else: - addr = addr.ipv6() - - if policy is not None and addr in policy: - if attempt: - attempt.failed() - LOG.info("Deleting Address {0} due to policy " - "violation".format( - address["address_readable"])) - - context.session.delete(address) - continue - if addr in cidr: - LOG.info("Marking Address {0} as " - "allocated".format( - address["address_readable"])) - updated_address = db_api.ip_address_update( - elevated, address, deallocated=False, - deallocated_at=None, - used_by_tenant_id=context.tenant_id, - allocated_at=timeutils.utcnow(), - port_id=port_id, - address_type=kwargs.get('address_type', - ip_types.FIXED)) - return [updated_address] - else: - if attempt: - attempt.failed() - # Make sure we never find it again - LOG.info("Address {0} isn't in the subnet " - "it claims to be in".format( - address["address_readable"])) - context.session.delete(address) - else: - if attempt: - attempt.failed() - LOG.info("Couldn't find any reallocatable addresses " - "given the criteria") - break + LOG.info("Address {0} is reallocated".format( + updated_address["address_readable"])) + return [updated_address] except Exception: if attempt: attempt.failed() diff --git a/quark/tests/functional/mysql/base.py b/quark/tests/functional/mysql/base.py index 82827c8..df6dde1 100644 --- a/quark/tests/functional/mysql/base.py +++ b/quark/tests/functional/mysql/base.py @@ -15,6 +15,10 @@ class MySqlBaseFunctionalTest(test_base.TestBase): 'connection', 'mysql://root@localhost/quark_functional_tests', 'database') + cfg.CONF.set_override( + 'connection_debug', + '100', + 'database') def setUp(self): super(MySqlBaseFunctionalTest, self).setUp() diff --git a/quark/tests/functional/mysql/test_db_ip_reallocate.py b/quark/tests/functional/mysql/test_db_ip_reallocate.py new file mode 100644 index 0000000..4e4a863 --- /dev/null +++ b/quark/tests/functional/mysql/test_db_ip_reallocate.py @@ -0,0 +1,335 @@ +import datetime + +import netaddr +from oslo.utils import timeutils + +from quark.db import api as db_api +from quark.plugin_modules import ip_policies +from quark.tests.functional.mysql.base import MySqlBaseFunctionalTest + + +class IPReallocateMixin(object): + REUSE_AFTER = 300 + + def insert_network(self): + tenant_id = "foobar" + network = {"tenant_id": tenant_id} + network_db = db_api.network_create(self.context, **network) + self.context.session.flush() + return network_db + + def insert_subnet(self, network_db, cidr, do_not_use=False): + subnet = {"network": network_db, + "cidr": cidr, + "ip_version": netaddr.IPNetwork(cidr).version, + "do_not_use": do_not_use} + subnet_db = db_api.subnet_create(self.context, **subnet) + self.context.session.flush() + return subnet_db + + def insert_ip_address(self, ip_address, network_db, subnet_db): + ip_address_db = db_api.ip_address_create( + self.context, + address=ip_address, + version=ip_address.version, + subnet_id=subnet_db["id"] if subnet_db else None, + network_id=network_db["id"]) + + ip_address_db["_deallocated"] = True + ip_address_db["deallocated_at"] = ( + timeutils.utcnow() - datetime.timedelta(seconds=self.REUSE_AFTER)) + self.context.session.add(ip_address_db) + self.context.session.flush() + + return ip_address_db + + def insert_transaction(self): + with self.context.session.begin(): + transaction = db_api.transaction_create(self.context) + return transaction + + def default_case(self): + self.network_db = self.insert_network() + self.subnet_v4_db = self.insert_subnet( + self.network_db, "192.168.0.0/24") + self.subnet_v6_db = self.insert_subnet( + self.network_db, "ffcc::/7") + self.ip_address_v4 = netaddr.IPAddress("192.168.0.1") + self.insert_ip_address(self.ip_address_v4, self.network_db, + self.subnet_v4_db) + self.ip_address_v6 = netaddr.IPAddress("ffcc::1") + self.insert_ip_address(self.ip_address_v6, self.network_db, + self.subnet_v6_db) + self.transaction = self.insert_transaction() + + def insert_default_ip_policy(self, subnet_db): + cidrs = [] + ip_policies.ensure_default_policy(cidrs, [subnet_db]) + subnet_db["ip_policy"] = db_api.ip_policy_create( + self.context, exclude=cidrs) + self.context.session.add(subnet_db) + self.context.session.flush() + + +class QuarkIPReallocateFunctionalTest(MySqlBaseFunctionalTest, + IPReallocateMixin): + def setUp(self): + super(QuarkIPReallocateFunctionalTest, self).setUp() + self.default_case() + + def test_normal_v4(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + def test_normal_v6(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 6, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + def test_ip_address_specified_deallocated_None(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": None, + "ip_address": self.ip_address_v4, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + def test_subnet_ids_specified(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": [self.subnet_v4_db["id"]] + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + def test_reuse_after_not_time_yet(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER * 2, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertFalse(reallocated) + + def test_normal_one_of_multiple_potential_ip_addresses(self): + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": None, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + +class QuarkIPReallocateFindTest(MySqlBaseFunctionalTest, IPReallocateMixin): + def test_normal_v4(self): + self.default_case() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertEqual(updated_address["address"], + int(self.ip_address_v4.ipv6())) + + def test_normal_v6(self): + self.default_case() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 6, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertEqual(updated_address["address"], + int(self.ip_address_v6.ipv6())) + + def test_address_not_found(self): + self.transaction = self.insert_transaction() + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertIsNone(updated_address) + + def test_subnet_null(self): + self.network_db = self.insert_network() + self.subnet_v4_db = self.insert_subnet( + self.network_db, "192.168.0.0/24") + self.ip_address_v4 = netaddr.IPAddress("192.168.0.1") + self.insert_ip_address(self.ip_address_v4, self.network_db, None) + self.transaction = self.insert_transaction() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertIsNone(updated_address) + + def test_subnet_do_not_use(self): + self.network_db = self.insert_network() + self.subnet_v4_db = self.insert_subnet( + self.network_db, "192.168.0.0/24", do_not_use=True) + self.ip_address_v4 = netaddr.IPAddress("192.168.0.1") + self.insert_ip_address(self.ip_address_v4, self.network_db, + self.subnet_v4_db) + self.transaction = self.insert_transaction() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertIsNone(updated_address) + + def test_policy_violation(self): + self.network_db = self.insert_network() + self.subnet_v4_db = self.insert_subnet( + self.network_db, "192.168.0.0/24") + self.ip_address_v4 = netaddr.IPAddress("192.168.0.0") + ip_address_db = self.insert_ip_address(self.ip_address_v4, + self.network_db, + self.subnet_v4_db) + self.transaction = self.insert_transaction() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + self.insert_default_ip_policy(self.subnet_v4_db) + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertIsNone(updated_address) + + self.context.session.flush() + self.assertIsNone(db_api.ip_address_find(self.context, + id=ip_address_db.id, + scope=db_api.ONE)) + + def test_address_not_in_cidr(self): + self.network_db = self.insert_network() + self.subnet_v4_db = self.insert_subnet( + self.network_db, "192.168.0.0/24") + self.ip_address_v4 = netaddr.IPAddress("192.168.1.1") + ip_address_db = self.insert_ip_address(self.ip_address_v4, + self.network_db, + self.subnet_v4_db) + self.transaction = self.insert_transaction() + ip_kwargs = { + "network_id": self.network_db["id"], + "reuse_after": self.REUSE_AFTER, + "deallocated": True, + "ip_address": None, + "version": 4, + "subnet_id": None + } + reallocated = db_api.ip_address_reallocate( + self.context, + {"transaction_id": self.transaction.id}, + **ip_kwargs) + self.assertTrue(reallocated) + + updated_address = db_api.ip_address_reallocate_find( + self.context, self.transaction.id) + self.assertIsNone(updated_address) + + self.context.session.flush() + self.assertIsNone(db_api.ip_address_find(self.context, + id=ip_address_db.id, + scope=db_api.ONE)) diff --git a/quark/tests/functional/mysql/test_sqlalchemy_adapter.py b/quark/tests/functional/mysql/test_sqlalchemy_adapter.py new file mode 100644 index 0000000..db63f51 --- /dev/null +++ b/quark/tests/functional/mysql/test_sqlalchemy_adapter.py @@ -0,0 +1,27 @@ +from quark.db import models +from quark.db import sqlalchemy_adapter as quark_sa +from quark.tests.functional.mysql.base import MySqlBaseFunctionalTest + + +class QuarkSqlAlchemyFunctionalTest(MySqlBaseFunctionalTest): + def test_mysql_limit_1(self): + notfoobar = "notfoobar" + ip1 = models.IPAddress(address=0, address_readable="0", + used_by_tenant_id=notfoobar) + ip2 = models.IPAddress(address=1, address_readable="1", + used_by_tenant_id=notfoobar) + self.context.session.add(ip1) + self.context.session.add(ip2) + self.context.session.flush() + + query = self.context.session.query(models.IPAddress) + row_count = quark_sa.update(query, + dict(used_by_tenant_id="foobar"), + update_args={"mysql_limit": 1}) + self.assertEqual(row_count, 1) + + self.context.session.refresh(ip1) + self.context.session.refresh(ip2) + self.assertEqual(sum((ip1["used_by_tenant_id"] == notfoobar, + ip2["used_by_tenant_id"] == notfoobar)), + 1) diff --git a/quark/tests/functional/mysql/test_transaction.py b/quark/tests/functional/mysql/test_transaction.py new file mode 100644 index 0000000..b9c1426 --- /dev/null +++ b/quark/tests/functional/mysql/test_transaction.py @@ -0,0 +1,15 @@ +from quark.db import api as quark_db_api +from quark.tests.functional.mysql.base import MySqlBaseFunctionalTest + + +class QuarkTransactionFunctionalTest(MySqlBaseFunctionalTest): + def test_transaction_id(self): + with self.context.session.begin(): + transaction = quark_db_api.transaction_create(self.context) + self.assertEqual(transaction.id, 1) + with self.context.session.begin(): + transaction = quark_db_api.transaction_create(self.context) + self.assertEqual(transaction.id, 2) + with self.context.session.begin(): + transaction = quark_db_api.transaction_create(self.context) + self.assertEqual(transaction.id, 3) diff --git a/quark/tests/test_ipam.py b/quark/tests/test_ipam.py index 801f401..dc52282 100644 --- a/quark/tests/test_ipam.py +++ b/quark/tests/test_ipam.py @@ -489,13 +489,16 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): if not addresses: addresses = [None, None] with contextlib.nested( + mock.patch("quark.db.api.ip_address_reallocate"), + mock.patch("quark.db.api.ip_address_reallocate_find"), mock.patch("quark.db.api.ip_address_find"), mock.patch("quark.db.api.subnet_find_ordered_by_most_full"), mock.patch("quark.db.api.subnet_find"), mock.patch("quark.db.api.subnet_update_next_auto_assign_ip"), mock.patch("quark.db.api.subnet_update_set_full"), mock.patch("sqlalchemy.orm.session.Session.refresh") - ) as (addr_find, subnet_alloc_find, subnet_find, subnet_update, + ) as (addr_realloc, addr_realloc_find, addr_find, + subnet_alloc_find, subnet_find, subnet_update, subnet_set_full, refresh): addr_mods = [] sub_mods = [] @@ -515,7 +518,9 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): sub_mod_list.append(sub) sub_mods.append(sub_mod_list) - addr_find.side_effect = addr_mods + addr_realloc.side_effect = addr_mods[:1] + addr_realloc_find.side_effect = addr_mods[:1] + addr_find.side_effect = addr_mods[1:] if sub_mods and len(sub_mods[0]): subnet_find.return_value = [sub_mods[0][0][0]] subnet_alloc_find.side_effect = sub_mods @@ -532,7 +537,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): refresh.side_effect = refresh_mock subnet_set_full.side_effect = set_full_mock - yield + yield addr_realloc def test_allocate_new_ip_address_unmarked_negative_one_full_subnet(self): net1 = netaddr.IPNetwork("0.0.0.0/24") @@ -663,14 +668,16 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): last_ip=last, next_auto_assign_ip=first) with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, mac_address=mac_address) self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], target_ip) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(address[1]["address"], expected_v6.value) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -693,7 +700,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address["version"] = 4 address["subnet"] = models.Subnet(cidr="0.0.0.0/24") with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, subnets=[subnet4['id']], @@ -701,7 +708,9 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(address[1]["address"], expected_v6.value) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -718,7 +727,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address["version"] = 4 address["subnet"] = models.Subnet(cidr="0.0.0.0/24") with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, segment_id="cell01", @@ -726,7 +735,9 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(address[1]["address"], expected_v6.value) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -744,13 +755,15 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address["version"] = 4 address["subnet"] = models.Subnet(cidr="0.0.0.0/24") with self._stubs(subnets=[[]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0) self.assertEqual(len(address), 1) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") def test_reallocate_deallocated_v6_ip(self): subnet4 = dict(id=1, first_ip=0, last_ip=255, @@ -762,13 +775,16 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address["version"] = 6 address["subnet"] = models.Subnet(cidr="::ffff:0:0/96") with self._stubs(subnets=[[(subnet4, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: addresses = [] self.ipam.allocate_ip_address(self.context, addresses, 0, 0, 0) self.assertEqual(len(addresses), 2) self.assertEqual(addresses[0]["address"], address["address"]) self.assertEqual(addresses[0]["version"], 6) - self.assertEqual(addresses[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") + self.assertEqual(addresses[1]["address"], netaddr.IPAddress('::ffff:0.0.0.1').value) self.assertEqual(addresses[1]["version"], 4) @@ -789,7 +805,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): mac["address"] = netaddr.EUI("AA:BB:CC:DD:EE:FF") with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, mac_address=mac) @@ -797,7 +813,10 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") + self.assertEqual(address[1]["address"], generated_v6.value) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -840,13 +859,16 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address["version"] = 6 address["subnet"] = models.Subnet(cidr="::ffff:0:0/96") with self._stubs(subnets=[[(subnet4, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: addresses = [] self.ipam.allocate_ip_address(self.context, addresses, 0, 0, 0) self.assertEqual(len(addresses), 2) self.assertEqual(addresses[0]["address"], str(self.v46_val)) self.assertEqual(addresses[0]["version"], 6) - self.assertEqual(addresses[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") + self.assertEqual(addresses[1]["address"], netaddr.IPAddress("::ffff:0.0.0.1").value) self.assertEqual(addresses[1]["version"], 4) @@ -866,14 +888,17 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest): address2["version"] = 6 address2["subnet"] = models.Subnet(cidr="::ffff:0:0/96") with self._stubs(subnets=[[(subnet6, 1)]], - addresses=[address1, address2]): + addresses=[address1, address2]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, mac_address=mac_address) self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") + self.assertEqual(address[1]["address"], address2["address"]) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -893,13 +918,20 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest): addresses = [None, None] self.context.session.add = mock.Mock() with contextlib.nested( + mock.patch("quark.db.api.ip_address_reallocate"), + mock.patch("quark.db.api.ip_address_reallocate_find"), mock.patch("quark.db.api.ip_address_find"), mock.patch("quark.db.api.subnet_find_ordered_by_most_full"), mock.patch("quark.db.api.subnet_update_next_auto_assign_ip"), mock.patch("quark.db.api.subnet_update_set_full"), mock.patch("sqlalchemy.orm.session.Session.refresh") - ) as (addr_find, subnet_find, subnet_update, subnet_set_full, refresh): - addr_find.side_effect = [ip_helper(a) for a in addresses] + ) as (addr_realloc, addr_realloc_find, addr_find, + subnet_find, subnet_update, subnet_set_full, refresh): + addrs = [ip_helper(a) for a in addresses] + addr_realloc.side_effect = addrs[:1] + addr_realloc_find.side_effect = addrs[:1] + addr_find.side_effect = addrs[1:] + sub_mods = [] for sub_list in subnets: sub_mod_list = [] @@ -927,7 +959,7 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest): refresh.side_effect = refresh_mock subnet_set_full.side_effect = set_full_mock - yield + yield addr_realloc def test_allocate_new_ip_address_two_empty_subnets(self): mac_address = 0 @@ -994,14 +1026,16 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest): address["version"] = 4 address["subnet"] = models.Subnet(cidr="0.0.0.0/24") with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, mac_address=mac_address) self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], 4) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(address[1]["address"], expected_v6.value) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -1015,13 +1049,15 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest): address["version"] = 6 address["subnet"] = models.Subnet(cidr="::ffff:0:0/96") with self._stubs(subnets=[[(subnet4, 0)]], - addresses=[address, None, None]): + addresses=[address, None, None]) as addr_realloc: addresses = [] self.ipam.allocate_ip_address(self.context, addresses, 0, 0, 0) self.assertEqual(len(addresses), 2) self.assertEqual(addresses[0]["address"], address["address"]) self.assertEqual(addresses[0]["version"], 6) - self.assertEqual(addresses[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(addresses[1]["address"], netaddr.IPAddress("::ffff:0.0.0.1").value) self.assertEqual(addresses[1]["version"], 4) @@ -1043,14 +1079,16 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest): address2["subnet"] = subnet6["cidr"] with self._stubs(subnets=[[(subnet6, 0)]], - addresses=[address1, address2]): + addresses=[address1, address2]) as addr_realloc: address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, mac_address=mac_address) self.assertEqual(len(address), 2) self.assertEqual(address[0]["address"], self.v46_val) self.assertEqual(address[0]["version"], 4) - self.assertEqual(address[0]['address_type'], 'fixed') + self.assertEqual( + addr_realloc.call_args[0][1][models.IPAddress.address_type], + "fixed") self.assertEqual(address[1]["address"], address2["address"]) self.assertEqual(address[1]["version"], 6) self.assertEqual(address[1]['address_type'], 'fixed') @@ -1275,13 +1313,20 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): addresses = [None] self.context.session.add = mock.Mock() with contextlib.nested( + mock.patch("quark.db.api.ip_address_reallocate"), + mock.patch("quark.db.api.ip_address_reallocate_find"), mock.patch("quark.db.api.ip_address_find"), mock.patch("quark.db.api.subnet_find_ordered_by_most_full"), mock.patch("quark.db.api.subnet_update_next_auto_assign_ip"), mock.patch("quark.db.api.subnet_update_set_full"), mock.patch("sqlalchemy.orm.session.Session.refresh") - ) as (addr_find, subnet_find, subnet_update, subnet_set_full, refresh): - addr_find.side_effect = [ip_helper(a) for a in addresses] + ) as (addr_realloc, addr_realloc_find, addr_find, + subnet_find, subnet_update, subnet_set_full, refresh): + addrs = [ip_helper(a) for a in addresses] + addr_realloc.side_effect = addrs[:1] + addr_realloc_find.side_effect = addrs[:1] + addr_find.side_effect = addrs[1:] + if isinstance(subnets, list): subnet_find.return_value = [(subnet_helper(s), c) for s, c in subnets] @@ -1312,7 +1357,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): refresh.side_effect = refresh_mock subnet_set_full.side_effect = set_full_mock - yield addr_find + yield addr_realloc def test_allocate_new_ip_address_in_empty_subnet(self): subnet = dict(id=1, first_ip=0, last_ip=255, @@ -1336,7 +1381,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): ip_policy=None) with self._stubs(subnets=[(subnet, 0)], addresses=[None, None]) as ( - addr_find): + addr_realloc): address = [] ip_address = ["10.0.0.17"] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, @@ -1345,8 +1390,8 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): self.assertEqual(len(address), 1) self.assertEqual(netaddr.IPAddress(address[0]['address']), netaddr.IPAddress('::ffff:10.0.0.17')) - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" not in kwargs) def test_allocate_two_fixed_ipv4_addresses(self): @@ -1364,7 +1409,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): ip_policy=None) with self._stubs(subnets=([(subnet1, 0)], [(subnet2, 0)]), - addresses=[None, None]) as addr_find: + addresses=[None, None]) as addr_realloc: address = [] ip_addresses = ["192.168.0.17", "10.0.0.17"] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, @@ -1377,8 +1422,8 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): self.assertEqual(netaddr.IPAddress(address[1]['address']), netaddr.IPAddress('::ffff:10.0.0.17')) self.assertEqual(address[1]['version'], 4) - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" not in kwargs) def test_allocate_one_fixed_ipv6_address(self): @@ -1390,7 +1435,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): ip_policy=None) with self._stubs(subnets=[(subnet, 0)], addresses=[None, None]) as ( - addr_find): + addr_realloc): address = [] ip_address = ["feed::13"] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, @@ -1400,8 +1445,8 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): self.assertEqual(netaddr.IPAddress(address[0]['address']), netaddr.IPAddress('feed::13')) self.assertEqual(address[0]["version"], 6) - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" not in kwargs) def test_allocate_two_fixed_ipv6_addresses(self): @@ -1419,7 +1464,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): ip_policy=None) with self._stubs(subnets=([(subnet1, 0)], [(subnet2, 0)]), - addresses=[None, None]) as addr_find: + addresses=[None, None]) as addr_realloc: address = [] ip_addresses = ["feed::13", "feef::13"] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, @@ -1432,8 +1477,8 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): self.assertEqual(netaddr.IPAddress(address[1]['address']), netaddr.IPAddress('feef::13')) self.assertEqual(address[1]["version"], 6) - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" not in kwargs) def test_allocate_one_fixed_ipv6_and_one_fixed_ipv4_address(self): @@ -1451,7 +1496,7 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): ip_policy=None) with self._stubs(subnets=([(subnet1, 0)], [(subnet2, 0)]), - addresses=[None, None]) as addr_find: + addresses=[None, None]) as addr_realloc: address = [] ip_addresses = ["feed::13", "10.0.0.17"] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0, @@ -1464,8 +1509,8 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): self.assertEqual(netaddr.IPAddress(address[1]['address']), netaddr.IPAddress('::ffff:10.0.0.17')) self.assertEqual(address[1]["version"], 4) - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" not in kwargs) def test_allocate_ip_one_full_one_open_subnet(self): @@ -1482,15 +1527,15 @@ class QuarkNewIPAddressAllocation(QuarkIpamBaseTest): exclude=[models.IPPolicyCIDR(cidr="0.0.1.0/32")])) subnets = [(subnet1, 1), (subnet2, 0)] with self._stubs(subnets=subnets, addresses=[None, None]) as ( - addr_find): + addr_realloc): address = [] self.ipam.allocate_ip_address(self.context, address, 0, 0, 0) self.assertEqual(address[0]["address"], netaddr.IPAddress("::ffff:0.0.1.1").value) self.assertEqual(address[0]["subnet_id"], 2) self.assertEqual(address[0]['address_type'], 'fixed') - self.assertTrue(addr_find.called) - args, kwargs = addr_find.call_args + self.assertTrue(addr_realloc.called) + args, kwargs = addr_realloc.call_args self.assertTrue("deallocated" in kwargs) def test_allocate_ip_no_subnet_fails(self): @@ -1690,10 +1735,13 @@ class QuarkIPAddressAllocateDeallocated(QuarkIpamBaseTest): def _stubs(self, ip_find, subnet, address, addresses_found, sub_found=True): with contextlib.nested( + mock.patch("quark.db.api.ip_address_reallocate"), + mock.patch("quark.db.api.ip_address_reallocate_find"), mock.patch("quark.db.api.ip_address_find"), mock.patch("quark.db.api.ip_address_update"), mock.patch("quark.ipam.QuarkIpamANY._choose_available_subnet") - ) as (addr_find, addr_update, choose_subnet): + ) as (addr_realloc, addr_realloc_find, addr_find, addr_update, + choose_subnet): addr_mod = models.IPAddress(**address) subnet_mod = models.Subnet(**subnet) subnet_mod["next_auto_assign_ip"] = subnet["next_auto_assign_ip"] @@ -1708,6 +1756,8 @@ class QuarkIPAddressAllocateDeallocated(QuarkIpamBaseTest): else: addr_mods.append(None) + addr_realloc.side_effect = addr_mods + addr_realloc_find.side_effect = addr_mods addr_find.side_effect = addr_mods addr_update.return_value = addr_mod choose_subnet.return_value = [subnet_mod] @@ -1729,24 +1779,6 @@ class QuarkIPAddressAllocateDeallocated(QuarkIpamBaseTest): self.assertIsNotNone(ipaddress[0]['id']) self.assertFalse(choose_subnet.called) - def test_allocate_finds_deallocated_ip_out_of_range_deletes(self): - subnet = dict(id=1, ip_version=4, next_auto_assign_ip=2, - do_not_use=False, - cidr="0.0.0.0/29", ip_policy=None) - address = dict(id=1, address=254) - address2 = dict(id=1, address=1) - address["subnet"] = subnet - address2["subnet"] = subnet - addresses_found = [address, address2] - self.context.session.delete = mock.Mock() - with self._stubs(False, subnet, address, addresses_found, - sub_found=True): - addr = [] - self.ipam.allocate_ip_address(self.context, addr, 0, 0, 0) - self.assertTrue(self.context.session.delete.called) - self.assertEqual(len(addr), 1) - self.context.session.delete = mock.Mock() - def test_allocate_finds_no_deallocated_creates_new_ip(self): subnet = dict(id=1, ip_version=4, next_auto_assign_ip=2, cidr="0.0.0.0/24", first_ip=0, last_ip=255, diff --git a/tox.ini b/tox.ini index 697c442..32a8d5d 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ downloadcache = ~/cache/pip deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - flake8 --builtins=_ quark + flake8 --show-source --builtins=_ quark [testenv:cover] setenv = VIRTUAL_ENV={envdir}