Add timeout logic to googlecloud provider

octodns/provider/googlecloud.py

@@ -40,6 +40,8 @@ class GoogleCloudProvider(BaseProvider):
                     'NS', 'PTR', 'SPF', 'SRV', 'TXT'))
     SUPPORTS_GEO = False
 
+    CHANGE_LOOP_WAIT = 5
+
     def __init__(self, id, project=None, credentials_file=None,
                  *args, **kwargs):
 
@@ -101,13 +103,20 @@ class GoogleCloudProvider(BaseProvider):
                                    .format(class_name, change))
 
         gcloud_changes.create()
-        i = 1
-        while gcloud_changes.status != 'done':
-            self.log.debug("Waiting for changes to complete")
-            time.sleep(i)
+
+        for i in range(120):
             gcloud_changes.reload()
-            if i < 30:
-                i += 2
+            self.log.debug("Waiting for changes to complete")
+            # https://cloud.google.com/dns/api/v1/changes#resource
+            # status can be one of either "pending" or "done"
+            if gcloud_changes.status != 'pending':
+                break
+            self.log.debug("Waiting for changes to complete")
+            time.sleep(self.CHANGE_LOOP_WAIT)
+
+        if gcloud_changes.status != 'done':
+            raise RuntimeError("Timeout reached after {} seconds".format(
+                i * self.CHANGE_LOOP_WAIT))
 
     def _create_gcloud_zone(self, dns_name):
         """Creates a google cloud ManagedZone with dns_name, and zone named

tests/test_octodns_provider_googlecloud.py

@@ -206,7 +206,6 @@ class TestGoogleCloudProvider(TestCase):
         '''
         return GoogleCloudProvider(id=1, project="mock")
 
-    @patch('octodns.provider.googlecloud.time.sleep')
     @patch('octodns.provider.googlecloud.dns')
     def test___init__(self, *_):
         self.assertIsInstance(GoogleCloudProvider(id=1,
@@ -246,8 +245,7 @@ class TestGoogleCloudProvider(TestCase):
         gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.", "unit-tests")
         status_mock = Mock()
         return_values_for_status = iter(
-            ['', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
-             '', '', '', 'done'])
+            ["pending"] * 11 + ['done', 'done'])
         type(status_mock).status = PropertyMock(
             side_effect=return_values_for_status.next)
         gcloud_zone_mock.changes = Mock(return_value=status_mock)
@@ -291,6 +289,15 @@ class TestGoogleCloudProvider(TestCase):
                 'aa.unit.tests.', 'A', 9001, ['1.2.4.3'])
         ])
 
+        type(status_mock).status = "pending"
+
+        with self.assertRaises(RuntimeError):
+            provider.apply(Plan(
+                existing=[update_existing_r, delete_r],
+                desired=desired,
+                changes=changes
+            ))
+
         unsupported_change = Mock()
         unsupported_change.__len__ = Mock(return_value=1)
         type_mock = Mock()