model: fix timezone handling for Edm.DateTimeOffset

CHANGELOG.md

@@ -5,13 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
-- model: split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak
 
 ### Fixed
 - Fix Edm.Binary literal representation - Daniel Balko
+- Datetime support for Edm.DateTimeOffset - Reto Schneider
+- Disallow creation of non-UTC Edm.DateTime - Reto Schneider
+- Split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak
 
 ### Removed
-- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version.
+- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version. - Petr Hanak
 
 ## [1.7.1]
 

pyodata/v2/model.py

@@ -29,17 +29,6 @@
 TypeInfo = collections.namedtuple('TypeInfo', 'namespace name is_collection')
 
 
-def current_timezone():
-    """Default Timezone for Python datetime instances when parsed from
-       Edm.DateTime values and vice versa.
-
-       OData V2 does not mention Timezones in the documentation of
-       Edm.DateTime and UTC was chosen because it is universal.
-    """
-
-    return datetime.timezone.utc
-
-
 def modlog():
     return logging.getLogger(LOGGER_NAME)
 
@@ -210,7 +199,8 @@ def _build_types():
             Types.register_type(Typ('Edm.SByte', '0'))
             Types.register_type(Typ('Edm.String', '\'\'', EdmStringTypTraits()))
             Types.register_type(Typ('Edm.Time', 'time\'PT00H00M\''))
-            Types.register_type(Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00\''))
+            Types.register_type(
+                Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00Z\'', EdmDateTimeOffsetTypTraits()))
 
     @staticmethod
     def register_type(typ):
@@ -373,6 +363,40 @@ def from_literal(self, value):
         return base64.b64encode(binary).decode()
 
 
+def ms_since_epoch_to_datetime(value, tzinfo):
+    """Convert milliseconds since midnight 1.1.1970 to datetime"""
+    try:
+        # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function
+        return datetime.datetime(1970, 1, 1, tzinfo=tzinfo) + datetime.timedelta(milliseconds=int(value))
+    except (ValueError, OverflowError):
+        min_ticks = -62135596800000
+        max_ticks = 253402300799999
+        if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < min_ticks:
+            # Some service providers return false minimal date values.
+            # -62135596800000 is the lowest value PyOData could read.
+            # This workaround fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case.
+            return datetime.datetime(year=1, day=1, month=1, tzinfo=tzinfo)
+        if FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > max_ticks:
+            return datetime.datetime(year=9999, day=31, month=12, tzinfo=tzinfo)
+        raise PyODataModelError(f'Cannot decode datetime from value {value}. '
+                                f'Possible value range: {min_ticks} to {max_ticks}. '
+                                f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` '
+                                f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.')
+
+
+def parse_datetime_literal(value):
+    try:
+        return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
+    except ValueError:
+        try:
+            return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
+        except ValueError:
+            try:
+                return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M')
+            except ValueError:
+                raise PyODataModelError(f'Cannot decode datetime from value {value}.')
+
+
 class EdmDateTimeTypTraits(EdmPrefixedTypTraits):
     """Emd.DateTime traits
 
@@ -403,46 +427,48 @@ def to_literal(self, value):
             raise PyODataModelError(
                 f'Cannot convert value of type {type(value)} to literal. Datetime format is required.')
 
+        if value.tzinfo != datetime.timezone.utc:
+            raise PyODataModelError('Emd.DateTime accepts only UTC')
+
         # Sets timezone to none to avoid including timezone information in the literal form.
         return super(EdmDateTimeTypTraits, self).to_literal(value.replace(tzinfo=None).isoformat())
 
     def to_json(self, value):
         if isinstance(value, str):
             return value
 
+        if value.tzinfo != datetime.timezone.utc:
+            raise PyODataModelError('Emd.DateTime accepts only UTC')
+
         # Converts datetime into timestamp in milliseconds in UTC timezone as defined in ODATA specification
         # https://www.odata.org/documentation/odata-version-2-0/json-format/
-        return f'/Date({int(value.replace(tzinfo=current_timezone()).timestamp()) * 1000})/'
+        # See also: https://docs.python.org/3/library/datetime.html#datetime.datetime.timestamp
+        ticks = (value - datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(milliseconds=1)
+        return f'/Date({int(ticks)})/'
 
     def from_json(self, value):
 
         if value is None:
             return None
 
-        matches = re.match(r"^/Date\((.*)\)/$", value)
-        if not matches:
+        matches = re.match(r"^/Date\((?P<milliseconds_since_epoch>-?\d+)(?P<offset_in_minutes>[+-]\d+)?\)/$", value)
+        try:
+            milliseconds_since_epoch = matches.group('milliseconds_since_epoch')
+        except AttributeError:
             raise PyODataModelError(
-                f"Malformed value {value} for primitive Edm type. Expected format is /Date(value)/")
-        value = matches.group(1)
-
+                f"Malformed value {value} for primitive Edm.DateTime type."
+                " Expected format is /Date(<ticks>[±<offset>])/")
         try:
-            # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function
-            value = datetime.datetime(1970, 1, 1, tzinfo=current_timezone()) + datetime.timedelta(milliseconds=int(value))
-        except (ValueError, OverflowError):
-            if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < -62135596800000:
-                # Some service providers return false minimal date values.
-                # -62135596800000 is the lowest value PyOData could read.
-                # This workaroud fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case.
-                value = datetime.datetime(year=1, day=1, month=1, tzinfo=current_timezone())
-            elif FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > 253402300799999:
-                value = datetime.datetime(year=9999, day=31, month=12, tzinfo=current_timezone())
-            else:
-                raise PyODataModelError(f'Cannot decode datetime from value {value}. '
-                                        f'Possible value range: -62135596800000 to 253402300799999. '
-                                        f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` '
-                                        f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.')
-
-        return value
+            offset_in_minutes = int(matches.group('offset_in_minutes') or 0)
+            timedelta = datetime.timedelta(minutes=offset_in_minutes)
+        except ValueError:
+            raise PyODataModelError(
+                f"Malformed value {value} for primitive Edm.DateTime type."
+                " Expected format is /Date(<ticks>[±<offset>])/")
+        except AttributeError:
+            timedelta = datetime.timedelta()  # Missing offset is interpreted as UTC
+        # Might raise a PyODataModelError exception
+        return ms_since_epoch_to_datetime(milliseconds_since_epoch, datetime.timezone.utc) + timedelta
 
     def from_literal(self, value):
 
@@ -451,18 +477,85 @@ def from_literal(self, value):
 
         value = super(EdmDateTimeTypTraits, self).from_literal(value)
 
+        # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats
+        return parse_datetime_literal(value).replace(tzinfo=datetime.timezone.utc)
+
+
+class EdmDateTimeOffsetTypTraits(EdmPrefixedTypTraits):
+    """Emd.DateTimeOffset traits
+
+       Represents date and time, plus an offset in minutes from UTC, with values ranging from 12:00:00 midnight,
+       January 1, 1753 A.D. through 11:59:59 P.M, December 9999 A.D
+
+       Literal forms:
+       datetimeoffset'yyyy-mm-ddThh:mm[:ss]±ii:nn' (works for all time zones)
+       datetimeoffset'yyyy-mm-ddThh:mm[:ss]Z' (works only for UTC)
+       NOTE: Spaces are not allowed between datetimeoffset and quoted portion.
+       The datetime part is case-insensitive, the offset one is not.
+
+       Example 1: datetimeoffset'1970-01-01T00:00:01+00:30'
+        - /Date(1000+0030)/ (As DateTime, but with a 30 minutes timezone offset)
+       Example 1: datetimeoffset'1970-01-01T00:00:01-00:60'
+        - /Date(1000-0030)/ (As DateTime, but with a negative 60 minutes timezone offset)
+       https://blogs.sap.com/2017/01/05/date-and-time-in-sap-gateway-foundation/
+    """
+
+    def __init__(self):
+        super(EdmDateTimeOffsetTypTraits, self).__init__('datetimeoffset')
+
+    def to_literal(self, value):
+        """Convert python datetime representation to literal format"""
+
+        if not isinstance(value, datetime.datetime) or value.utcoffset() is None:
+            raise PyODataModelError(
+                f'Cannot convert value of type {type(value)} to literal. Datetime format including offset is required.')
+
+        return super(EdmDateTimeOffsetTypTraits, self).to_literal(value.isoformat())
+
+    def to_json(self, value):
+        # datetime.timestamp() does not work due to its limited precision
+        offset_in_minutes = int(value.utcoffset() / datetime.timedelta(minutes=1))
+        ticks = int((value - datetime.datetime(1970, 1, 1, tzinfo=value.tzinfo)) / datetime.timedelta(milliseconds=1))
+        return f'/Date({ticks}{offset_in_minutes:+05})/'
+
+    def from_json(self, value):
+        matches = re.match(r"^/Date\((?P<milliseconds_since_epoch>-?\d+)(?P<offset_in_minutes>[+-]\d+)\)/$", value)
         try:
-            value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
-        except ValueError:
-            try:
-                value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
-            except ValueError:
-                try:
-                    value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M')
-                except ValueError:
-                    raise PyODataModelError(f'Cannot decode datetime from value {value}.')
+            milliseconds_since_epoch = matches.group('milliseconds_since_epoch')
+            offset_in_minutes = int(matches.group('offset_in_minutes'))
+        except (ValueError, AttributeError):
+            raise PyODataModelError(
+                f"Malformed value {value} for primitive Edm.DateTimeOffset type."
+                " Expected format is /Date(<ticks>±<offset>)/")
+
+        tzinfo = datetime.timezone(datetime.timedelta(minutes=offset_in_minutes))
+        # Might raise a PyODataModelError exception
+        return ms_since_epoch_to_datetime(milliseconds_since_epoch, tzinfo)
+
+    def from_literal(self, value):
 
-        return value.replace(tzinfo=current_timezone())
+        if value is None:
+            return None
+
+        value = super(EdmDateTimeOffsetTypTraits, self).from_literal(value)
+
+        try:
+            # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats
+            if re.match(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', value, flags=re.ASCII | re.IGNORECASE):
+                datetime_part = value[:-1]
+                tz_info = datetime.timezone.utc
+            else:
+                match = re.match(r'(?P<datetime>.+)(?P<sign>[\\+-])(?P<hours>\d{2}):(?P<minutes>\d{2})',
+                                 value,
+                                 flags=re.ASCII)
+                datetime_part = match.group('datetime')
+                tz_offset = datetime.timedelta(hours=int(match.group('hours')),
+                                               minutes=int(match.group('minutes')))
+                tz_sign = -1 if match.group('sign') == '-' else 1
+                tz_info = datetime.timezone(tz_sign * tz_offset)
+            return parse_datetime_literal(datetime_part).replace(tzinfo=tz_info)
+        except (ValueError, AttributeError):
+            raise PyODataModelError(f'Cannot decode datetimeoffset from value {value}.')
 
 
 class EdmStringTypTraits(TypTraits):

tests/conftest.py

@@ -2,7 +2,7 @@
 import logging
 import os
 import pytest
-from pyodata.v2.model import schema_from_xml
+from pyodata.v2.model import schema_from_xml, Types
 
 
 def contents_of_fixtures_file(file_name):
@@ -129,3 +129,13 @@ def assert_logging_policy(mock_warning, *args):
 def assert_request_contains_header(headers, name, value):
     assert name in headers
     assert headers[name] == value
+
+
+@pytest.fixture
+def type_date_time():
+    return Types.from_name('Edm.DateTime')
+
+
+@pytest.fixture
+def type_date_time_offset():
+    return Types.from_name('Edm.DateTimeOffset')

tests/metadata.xml

@@ -59,6 +59,8 @@
                           sap:creatable="false" sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
                 <Property Name="Date" Type="Edm.DateTime" Nullable="false" sap:label="Data" sap:creatable="false"
                           sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
+                <Property Name="DateTimeWithOffset" Type="Edm.DateTimeOffset" Nullable="true" sap:label="Data"
+                          sap:creatable="false" sap:updatable="true" sap:sortable="true" sap:filterable="true"/>
                 <Property Name="Value" Type="Edm.Double" Nullable="false" sap:unicode="false" sap:label="Data"
                           sap:creatable="false" sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
             </EntityType>