diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index f87538d..80e9603 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -80,11 +80,48 @@ class Ns1Client(object): self._datasource = client.datasource() self._datafeed = client.datafeed() - self._datasource_id = None + self.reset_caches() + def reset_caches(self): + self._datasource_id = None self._feeds_for_monitors = None self._monitors_cache = None self._notifylists_cache = None + self._zones_cache = {} + self._records_cache = {} + + def update_record_cache(func): + def call(self, zone, domain, _type, **params): + if zone in self._zones_cache: + # remove record's zone from cache + del self._zones_cache[zone] + + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + + if _type in cached: + # remove record from cache + del cached[_type] + + # write record to cache if its not a delete + new_record = func(self, zone, domain, _type, **params) + if new_record: + cached[_type] = new_record + + return new_record + + return call + + def read_or_set_record_cache(func): + def call(self, zone, domain, _type): + cached = self._records_cache.setdefault(zone, {}) \ + .setdefault(domain, {}) + if _type not in cached: + cached[_type] = func(self, zone, domain, _type) + + return cached[_type] + + return call @property def datasource_id(self): @@ -192,23 +229,30 @@ class Ns1Client(object): def notifylists_list(self): return self._try(self._notifylists.list) + @update_record_cache def records_create(self, zone, domain, _type, **params): return self._try(self._records.create, zone, domain, _type, **params) + @update_record_cache def records_delete(self, zone, domain, _type): return self._try(self._records.delete, zone, domain, _type) + @read_or_set_record_cache def records_retrieve(self, zone, domain, _type): return self._try(self._records.retrieve, zone, domain, _type) + @update_record_cache def records_update(self, zone, domain, _type, **params): return self._try(self._records.update, zone, domain, _type, **params) def zones_create(self, name): - return self._try(self._zones.create, name) + self._zones_cache[name] = self._try(self._zones.create, name) + return self._zones_cache[name] def zones_retrieve(self, name): - return self._try(self._zones.retrieve, name) + if name not in self._zones_cache: + self._zones_cache[name] = self._try(self._zones.retrieve, name) + return self._zones_cache[name] def _try(self, method, *args, **kwargs): tries = self.retry_count diff --git a/tests/test_octodns_processor_acme.py b/tests/test_octodns_processor_acme.py index c927608..02177f7 100644 --- a/tests/test_octodns_processor_acme.py +++ b/tests/test_octodns_processor_acme.py @@ -71,7 +71,6 @@ class TestAcmeMangingProcessor(TestCase): ], sorted([r.name for r in got.records])) managed = None for record in got.records: - print(record.name) if record.name.endswith('managed'): managed = record break diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 6243348..1d3f49b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -197,7 +197,13 @@ class TestNs1Provider(TestCase): def test_populate(self, zone_retrieve_mock, record_retrieve_mock): provider = Ns1Provider('test', 'api-key') + def reset(): + provider._client.reset_caches() + zone_retrieve_mock.reset_mock() + record_retrieve_mock.reset_mock() + # Bad auth + reset() zone_retrieve_mock.side_effect = AuthException('unauthorized') zone = Zone('unit.tests.', []) with self.assertRaises(AuthException) as ctx: @@ -205,7 +211,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # General error - zone_retrieve_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = ResourceException('boom') zone = Zone('unit.tests.', []) with self.assertRaises(ResourceException) as ctx: @@ -214,7 +220,7 @@ class TestNs1Provider(TestCase): self.assertEquals(('unit.tests',), zone_retrieve_mock.call_args[0]) # Non-existent zone doesn't populate anything - zone_retrieve_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone = Zone('unit.tests.', []) @@ -224,8 +230,7 @@ class TestNs1Provider(TestCase): self.assertFalse(exists) # Existing zone w/o records - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "geo.unit.tests", @@ -255,8 +260,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Existing zone w/records - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ "domain": "geo.unit.tests", @@ -286,8 +290,7 @@ class TestNs1Provider(TestCase): 'geo.unit.tests', 'A')]) # Test skipping unsupported record type - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ 'type': 'UNSUPPORTED', @@ -340,19 +343,21 @@ class TestNs1Provider(TestCase): self.assertEquals(expected_n, len(plan.changes)) self.assertTrue(plan.exists) + def reset(): + provider._client.reset_caches() + record_retrieve_mock.reset_mock() + zone_create_mock.reset_mock() + zone_retrieve_mock.reset_mock() + # Fails, general error - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: provider.apply(plan) self.assertEquals(zone_retrieve_mock.side_effect, ctx.exception) # Fails, bad auth - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') zone_create_mock.side_effect = AuthException('unauthorized') @@ -361,9 +366,7 @@ class TestNs1Provider(TestCase): self.assertEquals(zone_create_mock.side_effect, ctx.exception) # non-existent zone, create - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() zone_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -395,9 +398,7 @@ class TestNs1Provider(TestCase): ]) # Update & delete - zone_retrieve_mock.reset_mock() - record_retrieve_mock.reset_mock() - zone_create_mock.reset_mock() + reset() ns1_zone = { 'records': self.ns1_records + [{ @@ -937,11 +938,14 @@ class TestNs1ProviderDynamic(TestCase): 'mon-id': 'feed-id', } + def reset(): + feed_create_mock.reset_mock() + monitor_create_mock.reset_mock() + monitor_gen_mock.reset_mock() + monitors_update_mock.reset_mock() + # No existing monitor - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor_gen_mock.side_effect = [{'key': 'value'}] monitor_create_mock.side_effect = [('mon-id', 'feed-id')] value = '1.2.3.4' @@ -955,10 +959,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_not_called() # Existing monitor that doesn't need updates - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id', 'key': 'value', @@ -975,10 +976,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_not_called() # Existing monitor that doesn't need updates, but is missing its feed - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id2', 'key': 'value', @@ -996,10 +994,7 @@ class TestNs1ProviderDynamic(TestCase): feed_create_mock.assert_has_calls([call(monitor)]) # Existing monitor that needs updates - monitor_gen_mock.reset_mock() - monitor_create_mock.reset_mock() - monitors_update_mock.reset_mock() - feed_create_mock.reset_mock() + reset() monitor = { 'id': 'mon-id', 'key': 'value', @@ -1033,11 +1028,14 @@ class TestNs1ProviderDynamic(TestCase): 'mon-id': 'feed-id', } + def reset(): + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + monitors_for_mock.reset_mock() + notifylists_delete_mock.reset_mock() + # No active monitors and no existing, nothing will happen - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{}] record = self.record() provider._monitors_gc(record) @@ -1047,10 +1045,7 @@ class TestNs1ProviderDynamic(TestCase): notifylists_delete_mock.assert_not_called() # No active monitors and one existing, delete all the things - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1070,10 +1065,7 @@ class TestNs1ProviderDynamic(TestCase): notifylists_delete_mock.assert_has_calls([call('nl-id')]) # Same existing, this time in active list, should be noop - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1088,10 +1080,7 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor w/o a feed, and another monitor that's left alone # b/c it's active - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'x': { 'id': 'mon-id', @@ -1120,10 +1109,7 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor w/o a notifylist, generally shouldn't happen, but # code should handle it just in case someone gets clicky in the UI - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() monitors_for_mock.side_effect = [{ 'y': { 'id': 'mon-id2', @@ -1148,11 +1134,8 @@ class TestNs1ProviderDynamic(TestCase): # Non-active monitor with a shared notifylist, monitor deleted, but # notifylist is left alone + reset() provider.shared_notifylist = True - monitors_for_mock.reset_mock() - datafeed_delete_mock.reset_mock() - monitors_delete_mock.reset_mock() - notifylists_delete_mock.reset_mock() monitors_for_mock.side_effect = [{ 'y': { 'id': 'mon-id2', @@ -1304,6 +1287,7 @@ class TestNs1ProviderDynamic(TestCase): # provider._params_for_A() calls provider._monitors_for() and # provider._monitor_sync(). Mock their return values so that we don't # make NS1 API calls during tests + provider._client.reset_caches() monitors_for_mock.reset_mock() monitor_sync_mock.reset_mock() monitors_for_mock.side_effect = [{ @@ -1934,34 +1918,35 @@ class TestNs1ProviderDynamic(TestCase): desired = Zone('unit.tests.', []) + def reset(): + monitors_for_mock.reset_mock() + provider._client.reset_caches() + records_retrieve_mock.reset_mock() + zones_retrieve_mock.reset_mock() + # Empty zone and no changes - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() extra = provider._extra_changes(desired, []) self.assertFalse(extra) monitors_for_mock.assert_not_called() # Non-existent zone. No changes - monitors_for_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') - records_retrieve_mock.reset_mock() extra = provider._extra_changes(desired, []) self.assertFalse(extra) # Unexpected exception message - zones_retrieve_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = ResourceException('boom') with self.assertRaises(ResourceException) as ctx: extra = provider._extra_changes(desired, []) self.assertEquals(zones_retrieve_mock.side_effect, ctx.exception) # Simple record, ignored, filter update lookups ignored - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() zones_retrieve_mock.side_effect = \ ResourceException('server error: zone not found') @@ -2006,9 +1991,7 @@ class TestNs1ProviderDynamic(TestCase): desired.add_record(dynamic) # untouched, but everything in sync so no change needed - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() # Generate what we expect to have gend = provider._monitor_gen(dynamic, '1.2.3.4') gend.update({ @@ -2026,9 +2009,7 @@ class TestNs1ProviderDynamic(TestCase): # If we don't have a notify list we're broken and we'll expect to see # an Update - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() del gend['notify_list'] monitors_for_mock.side_effect = [{ '1.2.3.4': gend, @@ -2042,9 +2023,7 @@ class TestNs1ProviderDynamic(TestCase): # Add notify_list back and change the healthcheck protocol, we'll still # expect to see an update - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() gend['notify_list'] = 'xyz' dynamic._octodns['healthcheck']['protocol'] = 'HTTPS' del gend['notify_list'] @@ -2059,9 +2038,7 @@ class TestNs1ProviderDynamic(TestCase): monitors_for_mock.assert_has_calls([call(dynamic)]) # If it's in the changed list, it'll be ignored - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() extra = provider._extra_changes(desired, [update]) self.assertFalse(extra) monitors_for_mock.assert_not_called() @@ -2069,9 +2046,7 @@ class TestNs1ProviderDynamic(TestCase): # Test changes in filters # No change in filters - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2088,9 +2063,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertFalse(extra) # filters need an update - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2107,9 +2080,7 @@ class TestNs1ProviderDynamic(TestCase): self.assertTrue(extra) # Mixed disabled in filters. Raise Ns1Exception - monitors_for_mock.reset_mock() - zones_retrieve_mock.reset_mock() - records_retrieve_mock.reset_mock() + reset() ns1_zone = { 'records': [{ "domain": "dyn.unit.tests", @@ -2234,12 +2205,14 @@ class TestNs1Client(TestCase): client = Ns1Client('dummy-key') # No retry required, just calls and is returned + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = ['foo'] self.assertEquals('foo', client.zones_retrieve('unit.tests')) zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # One retry required + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('boo', period=0), @@ -2249,6 +2222,7 @@ class TestNs1Client(TestCase): zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # Two retries required + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('boo', period=0), @@ -2258,6 +2232,7 @@ class TestNs1Client(TestCase): zone_retrieve_mock.assert_has_calls([call('unit.tests')]) # Exhaust our retries + client.reset_caches() zone_retrieve_mock.reset_mock() zone_retrieve_mock.side_effect = [ RateLimitException('first', period=0), @@ -2469,9 +2444,12 @@ class TestNs1Client(TestCase): notifylists_delete_mock): client = Ns1Client('dummy-key') - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + def reset(): + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + notifylists_list_mock.reset_mock() + + reset() notifylists_list_mock.side_effect = [{}] expected = { 'id': 'nl-id', @@ -2493,9 +2471,7 @@ class TestNs1Client(TestCase): ]) notifylists_delete_mock.assert_not_called() - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() client.notifylists_delete('nlid') notifylists_list_mock.assert_not_called() notifylists_create_mock.assert_not_called() @@ -2503,9 +2479,7 @@ class TestNs1Client(TestCase): # Delete again, this time with a cache item that needs cleaned out and # another that needs to be ignored - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() client._notifylists_cache = { 'another': { 'id': 'notid', @@ -2524,9 +2498,7 @@ class TestNs1Client(TestCase): # Only another left self.assertEquals(['another'], list(client._notifylists_cache.keys())) - notifylists_list_mock.reset_mock() - notifylists_create_mock.reset_mock() - notifylists_delete_mock.reset_mock() + reset() expected = ['one', 'two', 'three'] notifylists_list_mock.side_effect = [expected] nls = client.notifylists_list() @@ -2534,3 +2506,150 @@ class TestNs1Client(TestCase): notifylists_list_mock.assert_has_calls([call()]) notifylists_create_mock.assert_not_called() notifylists_delete_mock.assert_not_called() + + @patch('ns1.rest.records.Records.delete') + @patch('ns1.rest.records.Records.update') + @patch('ns1.rest.records.Records.create') + @patch('ns1.rest.records.Records.retrieve') + @patch('ns1.rest.zones.Zones.create') + @patch('ns1.rest.zones.Zones.delete') + @patch('ns1.rest.zones.Zones.retrieve') + def test_client_caching(self, zone_retrieve_mock, zone_delete_mock, + zone_create_mock, record_retrieve_mock, + record_create_mock, record_update_mock, + record_delete_mock): + client = Ns1Client('dummy-key') + + def reset(): + zone_retrieve_mock.reset_mock() + zone_delete_mock.reset_mock() + zone_create_mock.reset_mock() + record_retrieve_mock.reset_mock() + record_create_mock.reset_mock() + record_update_mock.reset_mock() + record_delete_mock.reset_mock() + # Testing caches so we don't reset those + + # Initial zone get fetches and caches + reset() + zone_retrieve_mock.side_effect = ['foo'] + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_has_calls([call('unit.tests')]) + self.assertEquals({ + 'unit.tests': 'foo', + }, client._zones_cache) + + # Subsequent zone get does not fetch and returns from cache + reset() + self.assertEquals('foo', client.zones_retrieve('unit.tests')) + zone_retrieve_mock.assert_not_called() + + # Zone create stores in cache + reset() + zone_create_mock.side_effect = ['bar'] + self.assertEquals('bar', client.zones_create('sub.unit.tests')) + zone_create_mock.assert_has_calls([call('sub.unit.tests')]) + self.assertEquals({ + 'sub.unit.tests': 'bar', + 'unit.tests': 'foo', + }, client._zones_cache) + + # Initial record get fetches and caches + reset() + record_retrieve_mock.side_effect = ['baz'] + self.assertEquals('baz', client.records_retrieve('unit.tests', + 'a.unit.tests', 'A')) + record_retrieve_mock.assert_has_calls([call('unit.tests', + 'a.unit.tests', 'A')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + } + } + }, client._records_cache) + + # Subsequent record get does not fetch and returns from cache + reset() + self.assertEquals('baz', client.records_retrieve('unit.tests', + 'a.unit.tests', 'A')) + record_retrieve_mock.assert_not_called() + + # Record create stores in cache + reset() + record_create_mock.side_effect = ['boo'] + self.assertEquals('boo', client.records_create('unit.tests', + 'aaaa.unit.tests', + 'AAAA', key='val')) + record_create_mock.assert_has_calls([call('unit.tests', + 'aaaa.unit.tests', 'AAAA', + key='val')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + }, + 'aaaa.unit.tests': { + 'AAAA': 'boo' + }, + } + }, client._records_cache) + + # Record delete removes from cache and removes zone + reset() + record_delete_mock.side_effect = [{}] + self.assertEquals({}, client.records_delete('unit.tests', + 'aaaa.unit.tests', + 'AAAA')) + record_delete_mock.assert_has_calls([call('unit.tests', + 'aaaa.unit.tests', 'AAAA')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': { + 'A': 'baz' + }, + 'aaaa.unit.tests': {}, + } + }, client._records_cache) + self.assertEquals({ + 'sub.unit.tests': 'bar', + }, client._zones_cache) + + # Delete the other record, no zone this time, record should still go + # away + reset() + record_delete_mock.side_effect = [{}] + self.assertEquals({}, client.records_delete('unit.tests', + 'a.unit.tests', 'A')) + record_delete_mock.assert_has_calls([call('unit.tests', 'a.unit.tests', + 'A')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': {}, + 'aaaa.unit.tests': {}, + } + }, client._records_cache) + self.assertEquals({ + 'sub.unit.tests': 'bar', + }, client._zones_cache) + + # Record update removes zone and caches result + record_update_mock.side_effect = ['done'] + self.assertEquals('done', client.records_update('sub.unit.tests', + 'aaaa.sub.unit.tests', + 'AAAA', key='val')) + record_update_mock.assert_has_calls([call('sub.unit.tests', + 'aaaa.sub.unit.tests', + 'AAAA', key='val')]) + self.assertEquals({ + 'unit.tests': { + 'a.unit.tests': {}, + 'aaaa.unit.tests': {}, + }, + 'sub.unit.tests': { + 'aaaa.sub.unit.tests': { + 'AAAA': 'done', + }, + } + }, client._records_cache) + self.assertEquals({}, client._zones_cache)