Merge pull request #70 from github/natrual-sorting-improvements

Natural sorting improvements

octodns/provider/yaml.py

@@ -26,16 +26,22 @@ class YamlProvider(BaseProvider):
         # The ttl to use for records when not specified in the data
         # (optional, default 3600)
         default_ttl: 3600
+        # Whether or not to enforce sorting order on the yaml config
+        # (optional, default True)
+        enforce_order: True
     '''
     SUPPORTS_GEO = True
 
-    def __init__(self, id, directory, default_ttl=3600, *args, **kwargs):
+    def __init__(self, id, directory, default_ttl=3600, enforce_order=True,
+                 *args, **kwargs):
         self.log = logging.getLogger('YamlProvider[{}]'.format(id))
-        self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d', id,
-                       directory, default_ttl)
+        self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, '
+                       'enforce_order=%d', id, directory, default_ttl,
+                       enforce_order)
         super(YamlProvider, self).__init__(id, *args, **kwargs)
         self.directory = directory
         self.default_ttl = default_ttl
+        self.enforce_order = enforce_order
 
     def populate(self, zone, target=False):
         self.log.debug('populate: zone=%s, target=%s', zone.name, target)
@@ -47,7 +53,7 @@ class YamlProvider(BaseProvider):
         before = len(zone.records)
         filename = join(self.directory, '{}yaml'.format(zone.name))
         with open(filename, 'r') as fh:
-            yaml_data = safe_load(fh)
+            yaml_data = safe_load(fh, enforce_order=self.enforce_order)
             if yaml_data:
                 for name, data in yaml_data.items():
                     if not isinstance(data, list):

octodns/yaml.py

@@ -5,25 +5,12 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
+from natsort import natsort_keygen
 from yaml import SafeDumper, SafeLoader, load, dump
 from yaml.constructor import ConstructorError
-import re
 
 
-# zero-padded sort, simplified version of
-# https://www.xormedia.com/natural-sort-order-with-zero-padding/
-_pad_re = re.compile('\d+')
-
-
-def _zero_pad(match):
-    return '{:04d}'.format(int(match.group(0)))
-
-
-def _zero_padded_numbers(s):
-    try:
-        int(s)
-    except ValueError:
-        return _pad_re.sub(lambda d: _zero_pad(d), s)
+_natsort_key = natsort_keygen()
 
 
 # Found http://stackoverflow.com/a/21912744 which guided me on how to hook in
@@ -34,7 +21,7 @@ class SortEnforcingLoader(SafeLoader):
         self.flatten_mapping(node)
         ret = self.construct_pairs(node)
         keys = [d[0] for d in ret]
-        if keys != sorted(keys, key=_zero_padded_numbers):
+        if keys != sorted(keys, key=_natsort_key):
             raise ConstructorError(None, None, "keys out of order: {}"
                                    .format(', '.join(keys)), node.start_mark)
         return dict(ret)
@@ -59,7 +46,7 @@ class SortingDumper(SafeDumper):
 
     def _representer(self, data):
         data = data.items()
-        data.sort(key=lambda d: _zero_padded_numbers(d[0]))
+        data.sort(key=lambda d: _natsort_key(d[0]))
         return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data)
 
 

requirements.txt

@@ -10,6 +10,7 @@ futures==3.0.5
 incf.countryutils==1.0
 ipaddress==1.0.18
 jmespath==0.9.0
+natsort==5.0.3
 nsone==0.9.10
 python-dateutil==2.6.0
 requests==2.13.0

setup.py

@@ -34,6 +34,7 @@ setup(
         'futures>=3.0.5',
         'incf.countryutils>=1.0',
         'ipaddress>=1.0.18',
+        'natsort>=5.0.3',
         'python-dateutil>=2.6.0',
         'requests>=2.13.0'
     ],

tests/test_octodns_provider_yaml.py

@@ -100,6 +100,12 @@ class TestYamlProvider(TestCase):
         with self.assertRaises(ConstructorError):
             source.populate(zone)
 
+        source = YamlProvider('test', join(dirname(__file__), 'config'),
+                              enforce_order=False)
+        # no exception
+        source.populate(zone)
+        self.assertEqual(2, len(zone.records))
+
     def test_subzone_handling(self):
         source = YamlProvider('test', join(dirname(__file__), 'config'))
 

tests/test_octodns_yaml.py

@@ -59,3 +59,12 @@ class TestYaml(TestCase):
         }, buf)
         self.assertEquals("---\n'*.1.1': 42\n'*.2.1': 44\n'*.11.1': 43\n",
                           buf.getvalue())
+
+        # hex sorting isn't ideal, not treated as hex, this make sure we don't
+        # change the behavior
+        buf = StringIO()
+        safe_dump({
+            '45a03129': 42,
+            '45a0392a': 43,
+        }, buf)
+        self.assertEquals("---\n45a0392a: 43\n45a03129: 42\n", buf.getvalue())