Merge pull request #119 in SOLNSC/lib-solutions-python from bugfix/ADDON-26054-upgrade-pyyaml-to-5.3.1-or-later to master

* commit 'ae145f8419fd5e6b4bff83657a7f392d80e9b643':
  ADDON-26054:Updating the pyyaml version from 5.2 to 5.3.1 for security issue (ID: CVE-2020-1747)

Author: splunk-pwu

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Splunk Solution Library for Python Changelog
 
+## Version 2.0.7
+* Updated `yaml` from 5.2 to 5.3.1 for security issue (ID: CVE-2020-1747) and fixed as part of ADDON-26054
+
 ## Version 2.0.6
 * Fixed server info `captain_info` method exception handling python2/3 compatibility issue
 

solnlib/__init__.py

@@ -60,4 +60,4 @@
            'user_access',
            'utils']
 
-__version__ = '2.0.6'
+__version__ = '2.0.7'

solnlib/packages/yamlpy2/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2019 Ingy döt Net
+Copyright (c) 2017-2020 Ingy döt Net
 Copyright (c) 2006-2016 Kirill Simonov
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of

solnlib/packages/yamlpy2/__init__.py

@@ -8,7 +8,7 @@
 from loader import *
 from dumper import *
 
-__version__ = '5.2'
+__version__ = '5.3.1'
 
 try:
     from cyaml import *
@@ -345,7 +345,7 @@ def add_constructor(tag, constructor, Loader=None):
     Constructor is a function that accepts a Loader instance
     and a node object and produces the corresponding Python object.
     """
-    if Loader == None:
+    if Loader is None:
         loader.Loader.add_constructor(tag, constructor)
         loader.FullLoader.add_constructor(tag, constructor)
         loader.UnsafeLoader.add_constructor(tag, constructor)
@@ -359,7 +359,7 @@ def add_multi_constructor(tag_prefix, multi_constructor, Loader=None):
     Multi-constructor accepts a Loader instance, a tag suffix,
     and a node object and produces the corresponding Python object.
     """
-    if Loader == None:
+    if Loader is None:
         loader.Loader.add_multi_constructor(tag_prefix, multi_constructor)
         loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor)
         loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor)

solnlib/packages/yamlpy2/constructor.py

@@ -18,6 +18,29 @@
 class ConstructorError(MarkedYAMLError):
     pass
 
+
+class timezone(datetime.tzinfo):
+    def __init__(self, offset):
+        self._offset = offset
+        seconds = abs(offset).total_seconds()
+        self._name = 'UTC%s%02d:%02d' % (
+            '-' if offset.days < 0 else '+',
+            seconds // 3600,
+            seconds % 3600 // 60
+        )
+
+    def tzname(self, dt=None):
+        return self._name
+
+    def utcoffset(self, dt=None):
+        return self._offset
+
+    def dst(self, dt=None):
+        return datetime.timedelta(0)
+
+    __repr__ = __str__ = tzname
+
+
 class BaseConstructor(object):
 
     yaml_constructors = {}
@@ -33,6 +56,14 @@ def check_data(self):
         # If there are more documents available?
         return self.check_node()
 
+    def check_state_key(self, key):
+        """Block special attributes/methods from being set in a newly created
+        object, to prevent user-controlled methods from being called during
+        deserialization"""
+        if self.get_state_keys_blacklist_regexp().match(key):
+            raise ConstructorError(None, None,
+                "blacklisted key '%s' in instance state found" % (key,), None)
+
     def get_data(self):
         # Construct and return the next document.
         if self.check_node():
@@ -74,7 +105,7 @@ def construct_object(self, node, deep=False):
             constructor = self.yaml_constructors[node.tag]
         else:
             for tag_prefix in self.yaml_multi_constructors:
-                if node.tag.startswith(tag_prefix):
+                if tag_prefix is not None and node.tag.startswith(tag_prefix):
                     tag_suffix = node.tag[len(tag_prefix):]
                     constructor = self.yaml_multi_constructors[tag_prefix]
                     break
@@ -293,7 +324,7 @@ def construct_yaml_binary(self, node):
             return str(value).decode('base64')
         except (binascii.Error, UnicodeEncodeError), exc:
             raise ConstructorError(None, None,
-                    "failed to decode base64 data: %s" % exc, node.start_mark) 
+                    "failed to decode base64 data: %s" % exc, node.start_mark)
 
     timestamp_regexp = re.compile(
             ur'''^(?P<year>[0-9][0-9][0-9][0-9])
@@ -320,22 +351,23 @@ def construct_yaml_timestamp(self, node):
         minute = int(values['minute'])
         second = int(values['second'])
         fraction = 0
+        tzinfo = None
         if values['fraction']:
             fraction = values['fraction'][:6]
             while len(fraction) < 6:
                 fraction += '0'
             fraction = int(fraction)
-        delta = None
         if values['tz_sign']:
             tz_hour = int(values['tz_hour'])
             tz_minute = int(values['tz_minute'] or 0)
             delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
             if values['tz_sign'] == '-':
                 delta = -delta
-        data = datetime.datetime(year, month, day, hour, minute, second, fraction)
-        if delta:
-            data -= delta
-        return data
+            tzinfo = timezone(delta)
+        elif values['tz']:
+            tzinfo = timezone(datetime.timedelta(0))
+        return datetime.datetime(year, month, day, hour, minute, second, fraction,
+                                 tzinfo=tzinfo)
 
     def construct_yaml_omap(self, node):
         # Note: we do not check for duplicate keys, because it's too
@@ -471,6 +503,16 @@ def construct_undefined(self, node):
         SafeConstructor.construct_undefined)
 
 class FullConstructor(SafeConstructor):
+    # 'extend' is blacklisted because it is used by
+    # construct_python_object_apply to add `listitems` to a newly generate
+    # python instance
+    def get_state_keys_blacklist(self):
+        return ['^extend$', '^__.*__$']
+
+    def get_state_keys_blacklist_regexp(self):
+        if not hasattr(self, 'state_keys_blacklist_regexp'):
+            self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')')
+        return self.state_keys_blacklist_regexp
 
     def construct_python_str(self, node):
         return self.construct_scalar(node).encode('utf-8')
@@ -497,7 +539,7 @@ def find_python_module(self, name, mark, unsafe=False):
             except ImportError, exc:
                 raise ConstructorError("while constructing a Python module", mark,
                         "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
-        if not name in sys.modules:
+        if name not in sys.modules:
             raise ConstructorError("while constructing a Python module", mark,
                     "module %r is not imported" % name.encode('utf-8'), mark)
         return sys.modules[name]
@@ -517,7 +559,7 @@ def find_python_name(self, name, mark, unsafe=False):
             except ImportError, exc:
                 raise ConstructorError("while constructing a Python object", mark,
                         "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
-        if not module_name in sys.modules:
+        if module_name not in sys.modules:
             raise ConstructorError("while constructing a Python object", mark,
                     "module %r is not imported" % module_name.encode('utf-8'), mark)
         module = sys.modules[module_name]
@@ -566,19 +608,24 @@ def make_python_instance(self, suffix, node,
         else:
             return cls(*args, **kwds)
 
-    def set_python_instance_state(self, instance, state):
+    def set_python_instance_state(self, instance, state, unsafe=False):
         if hasattr(instance, '__setstate__'):
             instance.__setstate__(state)
         else:
             slotstate = {}
             if isinstance(state, tuple) and len(state) == 2:
                 state, slotstate = state
             if hasattr(instance, '__dict__'):
+                if not unsafe and state:
+                    for key in state.keys():
+                        self.check_state_key(key)
                 instance.__dict__.update(state)
             elif state:
                 slotstate.update(state)
             for key, value in slotstate.items():
-                setattr(object, key, value)
+                if not unsafe:
+                    self.check_state_key(key)
+                setattr(instance, key, value)
 
     def construct_python_object(self, suffix, node):
         # Format:
@@ -699,6 +746,10 @@ def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False)
         return super(UnsafeConstructor, self).make_python_instance(
             suffix, node, args, kwds, newobj, unsafe=True)
 
+    def set_python_instance_state(self, instance, state):
+        return super(UnsafeConstructor, self).set_python_instance_state(
+            instance, state, unsafe=True)
+
 UnsafeConstructor.add_multi_constructor(
     u'tag:yaml.org,2002:python/object/apply:',
     UnsafeConstructor.construct_python_object_apply)

solnlib/packages/yamlpy2/loader.py

@@ -51,7 +51,7 @@ def __init__(self, stream):
 # UnsafeLoader is the same as Loader (which is and was always unsafe on
 # untrusted input). Use of either Loader or UnsafeLoader should be rare, since
 # FullLoad should be able to load almost all YAML safely. Loader is left intact
-# to ensure backwards compatability.
+# to ensure backwards compatibility.
 class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
 
     def __init__(self, stream):

solnlib/packages/yamlpy2/reader.py

@@ -139,7 +139,7 @@ def determine_encoding(self):
     if has_ucs4:
         NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]')
     else:
-        NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD]')
+        NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)')
     def check_printable(self, data):
         match = self.NON_PRINTABLE.search(data)
         if match:

solnlib/packages/yamlpy2/representer.py

@@ -3,11 +3,12 @@
     'RepresenterError']
 
 from error import *
+
 from nodes import *
 
 import datetime
 
-import sys, copy_reg, types
+import copy_reg, types
 
 class RepresenterError(YAMLError):
     pass

solnlib/packages/yamlpy2/scanner.py

@@ -332,7 +332,7 @@ def unwind_indent(self, column):
         ## }
         #if self.flow_level and self.indent > column:
         #    raise ScannerError(None, None,
-        #            "invalid intendation or unclosed '[' or '{'",
+        #            "invalid indentation or unclosed '[' or '{'",
         #            self.get_mark())
 
         # In the flow context, indentation is ignored. We make the scanner less
@@ -370,7 +370,7 @@ def fetch_stream_start(self):
 
     def fetch_stream_end(self):
 
-        # Set the current intendation to -1.
+        # Set the current indentation to -1.
         self.unwind_indent(-1)
 
         # Reset simple keys.
@@ -389,7 +389,7 @@ def fetch_stream_end(self):
 
     def fetch_directive(self):
 
-        # Set the current intendation to -1.
+        # Set the current indentation to -1.
         self.unwind_indent(-1)
 
         # Reset simple keys.
@@ -407,7 +407,7 @@ def fetch_document_end(self):
 
     def fetch_document_indicator(self, TokenClass):
 
-        # Set the current intendation to -1.
+        # Set the current indentation to -1.
         self.unwind_indent(-1)
 
         # Reset simple keys. Note that there could not be a block collection

solnlib/packages/yamlpy3/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2017-2019 Ingy döt Net
+Copyright (c) 2017-2020 Ingy döt Net
 Copyright (c) 2006-2016 Kirill Simonov
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of