diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cc509ed..8a9998f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] -- model: split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak ### Fixed - Fix Edm.Binary literal representation - Daniel Balko +- Datetime support for Edm.DateTimeOffset - Reto Schneider +- Disallow creation of non-UTC Edm.DateTime - Reto Schneider +- Split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak ### Removed -- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version. +- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version. - Petr Hanak ## [1.7.1] diff --git a/pyodata/v2/model.py b/pyodata/v2/model.py index e79cbee1..e3f2bd6f 100644 --- a/pyodata/v2/model.py +++ b/pyodata/v2/model.py @@ -29,17 +29,6 @@ TypeInfo = collections.namedtuple('TypeInfo', 'namespace name is_collection') -def current_timezone(): - """Default Timezone for Python datetime instances when parsed from - Edm.DateTime values and vice versa. - - OData V2 does not mention Timezones in the documentation of - Edm.DateTime and UTC was chosen because it is universal. - """ - - return datetime.timezone.utc - - def modlog(): return logging.getLogger(LOGGER_NAME) @@ -210,7 +199,8 @@ def _build_types(): Types.register_type(Typ('Edm.SByte', '0')) Types.register_type(Typ('Edm.String', '\'\'', EdmStringTypTraits())) Types.register_type(Typ('Edm.Time', 'time\'PT00H00M\'')) - Types.register_type(Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00\'')) + Types.register_type( + Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00Z\'', EdmDateTimeOffsetTypTraits())) @staticmethod def register_type(typ): @@ -373,6 +363,40 @@ def from_literal(self, value): return base64.b64encode(binary).decode() +def ms_since_epoch_to_datetime(value, tzinfo): + """Convert milliseconds since midnight 1.1.1970 to datetime""" + try: + # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function + return datetime.datetime(1970, 1, 1, tzinfo=tzinfo) + datetime.timedelta(milliseconds=int(value)) + except (ValueError, OverflowError): + min_ticks = -62135596800000 + max_ticks = 253402300799999 + if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < min_ticks: + # Some service providers return false minimal date values. + # -62135596800000 is the lowest value PyOData could read. + # This workaround fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case. + return datetime.datetime(year=1, day=1, month=1, tzinfo=tzinfo) + if FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > max_ticks: + return datetime.datetime(year=9999, day=31, month=12, tzinfo=tzinfo) + raise PyODataModelError(f'Cannot decode datetime from value {value}. ' + f'Possible value range: {min_ticks} to {max_ticks}. ' + f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` ' + f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.') + + +def parse_datetime_literal(value): + try: + return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f') + except ValueError: + try: + return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S') + except ValueError: + try: + return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M') + except ValueError: + raise PyODataModelError(f'Cannot decode datetime from value {value}.') + + class EdmDateTimeTypTraits(EdmPrefixedTypTraits): """Emd.DateTime traits @@ -403,6 +427,9 @@ def to_literal(self, value): raise PyODataModelError( f'Cannot convert value of type {type(value)} to literal. Datetime format is required.') + if value.tzinfo != datetime.timezone.utc: + raise PyODataModelError('Emd.DateTime accepts only UTC') + # Sets timezone to none to avoid including timezone information in the literal form. return super(EdmDateTimeTypTraits, self).to_literal(value.replace(tzinfo=None).isoformat()) @@ -410,39 +437,38 @@ def to_json(self, value): if isinstance(value, str): return value + if value.tzinfo != datetime.timezone.utc: + raise PyODataModelError('Emd.DateTime accepts only UTC') + # Converts datetime into timestamp in milliseconds in UTC timezone as defined in ODATA specification # https://www.odata.org/documentation/odata-version-2-0/json-format/ - return f'/Date({int(value.replace(tzinfo=current_timezone()).timestamp()) * 1000})/' + # See also: https://docs.python.org/3/library/datetime.html#datetime.datetime.timestamp + ticks = (value - datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(milliseconds=1) + return f'/Date({int(ticks)})/' def from_json(self, value): if value is None: return None - matches = re.match(r"^/Date\((.*)\)/$", value) - if not matches: + matches = re.match(r"^/Date\((?P-?\d+)(?P[+-]\d+)?\)/$", value) + try: + milliseconds_since_epoch = matches.group('milliseconds_since_epoch') + except AttributeError: raise PyODataModelError( - f"Malformed value {value} for primitive Edm type. Expected format is /Date(value)/") - value = matches.group(1) - + f"Malformed value {value} for primitive Edm.DateTime type." + " Expected format is /Date(])/") try: - # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function - value = datetime.datetime(1970, 1, 1, tzinfo=current_timezone()) + datetime.timedelta(milliseconds=int(value)) - except (ValueError, OverflowError): - if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < -62135596800000: - # Some service providers return false minimal date values. - # -62135596800000 is the lowest value PyOData could read. - # This workaroud fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case. - value = datetime.datetime(year=1, day=1, month=1, tzinfo=current_timezone()) - elif FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > 253402300799999: - value = datetime.datetime(year=9999, day=31, month=12, tzinfo=current_timezone()) - else: - raise PyODataModelError(f'Cannot decode datetime from value {value}. ' - f'Possible value range: -62135596800000 to 253402300799999. ' - f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` ' - f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.') - - return value + offset_in_minutes = int(matches.group('offset_in_minutes') or 0) + timedelta = datetime.timedelta(minutes=offset_in_minutes) + except ValueError: + raise PyODataModelError( + f"Malformed value {value} for primitive Edm.DateTime type." + " Expected format is /Date(])/") + except AttributeError: + timedelta = datetime.timedelta() # Missing offset is interpreted as UTC + # Might raise a PyODataModelError exception + return ms_since_epoch_to_datetime(milliseconds_since_epoch, datetime.timezone.utc) + timedelta def from_literal(self, value): @@ -451,18 +477,85 @@ def from_literal(self, value): value = super(EdmDateTimeTypTraits, self).from_literal(value) + # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats + return parse_datetime_literal(value).replace(tzinfo=datetime.timezone.utc) + + +class EdmDateTimeOffsetTypTraits(EdmPrefixedTypTraits): + """Emd.DateTimeOffset traits + + Represents date and time, plus an offset in minutes from UTC, with values ranging from 12:00:00 midnight, + January 1, 1753 A.D. through 11:59:59 P.M, December 9999 A.D + + Literal forms: + datetimeoffset'yyyy-mm-ddThh:mm[:ss]±ii:nn' (works for all time zones) + datetimeoffset'yyyy-mm-ddThh:mm[:ss]Z' (works only for UTC) + NOTE: Spaces are not allowed between datetimeoffset and quoted portion. + The datetime part is case-insensitive, the offset one is not. + + Example 1: datetimeoffset'1970-01-01T00:00:01+00:30' + - /Date(1000+0030)/ (As DateTime, but with a 30 minutes timezone offset) + Example 1: datetimeoffset'1970-01-01T00:00:01-00:60' + - /Date(1000-0030)/ (As DateTime, but with a negative 60 minutes timezone offset) + https://blogs.sap.com/2017/01/05/date-and-time-in-sap-gateway-foundation/ + """ + + def __init__(self): + super(EdmDateTimeOffsetTypTraits, self).__init__('datetimeoffset') + + def to_literal(self, value): + """Convert python datetime representation to literal format""" + + if not isinstance(value, datetime.datetime) or value.utcoffset() is None: + raise PyODataModelError( + f'Cannot convert value of type {type(value)} to literal. Datetime format including offset is required.') + + return super(EdmDateTimeOffsetTypTraits, self).to_literal(value.isoformat()) + + def to_json(self, value): + # datetime.timestamp() does not work due to its limited precision + offset_in_minutes = int(value.utcoffset() / datetime.timedelta(minutes=1)) + ticks = int((value - datetime.datetime(1970, 1, 1, tzinfo=value.tzinfo)) / datetime.timedelta(milliseconds=1)) + return f'/Date({ticks}{offset_in_minutes:+05})/' + + def from_json(self, value): + matches = re.match(r"^/Date\((?P-?\d+)(?P[+-]\d+)\)/$", value) try: - value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f') - except ValueError: - try: - value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S') - except ValueError: - try: - value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M') - except ValueError: - raise PyODataModelError(f'Cannot decode datetime from value {value}.') + milliseconds_since_epoch = matches.group('milliseconds_since_epoch') + offset_in_minutes = int(matches.group('offset_in_minutes')) + except (ValueError, AttributeError): + raise PyODataModelError( + f"Malformed value {value} for primitive Edm.DateTimeOffset type." + " Expected format is /Date(±)/") + + tzinfo = datetime.timezone(datetime.timedelta(minutes=offset_in_minutes)) + # Might raise a PyODataModelError exception + return ms_since_epoch_to_datetime(milliseconds_since_epoch, tzinfo) + + def from_literal(self, value): - return value.replace(tzinfo=current_timezone()) + if value is None: + return None + + value = super(EdmDateTimeOffsetTypTraits, self).from_literal(value) + + try: + # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats + if re.match(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', value, flags=re.ASCII | re.IGNORECASE): + datetime_part = value[:-1] + tz_info = datetime.timezone.utc + else: + match = re.match(r'(?P.+)(?P[\\+-])(?P\d{2}):(?P\d{2})', + value, + flags=re.ASCII) + datetime_part = match.group('datetime') + tz_offset = datetime.timedelta(hours=int(match.group('hours')), + minutes=int(match.group('minutes'))) + tz_sign = -1 if match.group('sign') == '-' else 1 + tz_info = datetime.timezone(tz_sign * tz_offset) + return parse_datetime_literal(datetime_part).replace(tzinfo=tz_info) + except (ValueError, AttributeError): + raise PyODataModelError(f'Cannot decode datetimeoffset from value {value}.') class EdmStringTypTraits(TypTraits): diff --git a/tests/conftest.py b/tests/conftest.py index 681401f6..4eb2d2c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import logging import os import pytest -from pyodata.v2.model import schema_from_xml +from pyodata.v2.model import schema_from_xml, Types def contents_of_fixtures_file(file_name): @@ -129,3 +129,13 @@ def assert_logging_policy(mock_warning, *args): def assert_request_contains_header(headers, name, value): assert name in headers assert headers[name] == value + + +@pytest.fixture +def type_date_time(): + return Types.from_name('Edm.DateTime') + + +@pytest.fixture +def type_date_time_offset(): + return Types.from_name('Edm.DateTimeOffset') diff --git a/tests/metadata.xml b/tests/metadata.xml index 50f573c6..5c4fe358 100644 --- a/tests/metadata.xml +++ b/tests/metadata.xml @@ -59,6 +59,8 @@ sap:creatable="false" sap:updatable="false" sap:sortable="true" sap:filterable="true"/> + diff --git a/tests/test_model_v2.py b/tests/test_model_v2.py index eb6756b0..19c98f63 100644 --- a/tests/test_model_v2.py +++ b/tests/test_model_v2.py @@ -1,12 +1,12 @@ """Tests for OData Model module""" # pylint: disable=line-too-long,too-many-locals,too-many-statements,invalid-name, too-many-lines, no-name-in-module, expression-not-assigned, pointless-statement import os -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from unittest.mock import patch import pytest from pyodata.v2.model import Schema, Typ, StructTypeProperty, Types, EntityType, EdmStructTypeSerializer, \ Association, AssociationSet, EndRole, AssociationSetEndRole, TypeInfo, MetadataBuilder, ParserError, PolicyWarning, \ - PolicyIgnore, Config, PolicyFatal, NullType, NullAssociation, current_timezone, StructType + PolicyIgnore, Config, PolicyFatal, NullType, NullAssociation, StructType, parse_datetime_literal from pyodata.exceptions import PyODataException, PyODataModelError, PyODataParserError from tests.conftest import assert_logging_policy import pyodata.v2.model @@ -472,30 +472,56 @@ def test_traits(): assert str(e_info.value).startswith("Malformed value '1234-56' for primitive") -def test_traits_datetime(): +@pytest.mark.parametrize('datetime_literal,expected', [ + ('2001-02-03T04:05:06.000007', datetime(2001, 2, 3, 4, 5, 6, microsecond=7)), + ('2001-02-03T04:05:06', datetime(2001, 2, 3, 4, 5, 6, 0)), + ('2001-02-03T04:05', datetime(2001, 2, 3, 4, 5, 0, 0)), +]) +def test_parse_datetime_literal(datetime_literal, expected): + assert parse_datetime_literal(datetime_literal) == expected + + +@pytest.mark.parametrize('illegal_input', [ + '2001-02-03T04:05:61', + '2001-02-03T04:61', + '2001-02-03T24:05', + '2001-02-32T04:05', + '2001-13-03T04:05', + '2001-00-03T04:05', + '01-02-03T04:05', + '2001-02-03T04:05.AAA', + '', +]) +def test_parse_datetime_literal_faulty(illegal_input): + with pytest.raises(PyODataModelError) as e_info: + parse_datetime_literal(f'{illegal_input}') + assert str(e_info.value).startswith(f'Cannot decode datetime from value {illegal_input}') + + +def test_traits_datetime(type_date_time): """Test Edm.DateTime traits""" - typ = Types.from_name('Edm.DateTime') - assert repr(typ.traits) == 'EdmDateTimeTypTraits' + type_date_time = Types.from_name('Edm.DateTime') + assert repr(type_date_time.traits) == 'EdmDateTimeTypTraits' # 1. direction Python -> OData - testdate = datetime(2005, 1, 28, 18, 30, 44, 123456, tzinfo=current_timezone()) - assert typ.traits.to_literal(testdate) == "datetime'2005-01-28T18:30:44.123456'" + testdate = datetime(2005, 1, 28, 18, 30, 44, 123456, tzinfo=timezone.utc) + assert type_date_time.traits.to_literal(testdate) == "datetime'2005-01-28T18:30:44.123456'" # without miliseconds part - testdate = datetime(2005, 1, 28, 18, 30, 44, 0) - assert typ.traits.to_literal(testdate) == "datetime'2005-01-28T18:30:44'" + testdate = datetime(2005, 1, 28, 18, 30, 44, 0, tzinfo=timezone.utc) + assert type_date_time.traits.to_literal(testdate) == "datetime'2005-01-28T18:30:44'" # serialization of invalid value with pytest.raises(PyODataModelError) as e_info: - typ.traits.to_literal('xyz') + type_date_time.traits.to_literal('xyz') assert str(e_info.value).startswith('Cannot convert value of type') # 2. direction Literal -> python # parsing full representation - testdate = typ.traits.from_literal("datetime'1976-11-23T03:33:06.654321'") + testdate = type_date_time.traits.from_literal("datetime'1976-11-23T03:33:06.654321'") assert testdate.year == 1976 assert testdate.month == 11 assert testdate.day == 23 @@ -503,36 +529,36 @@ def test_traits_datetime(): assert testdate.minute == 33 assert testdate.second == 6 assert testdate.microsecond == 654321 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing without miliseconds - testdate = typ.traits.from_literal("datetime'1976-11-23T03:33:06'") + testdate = type_date_time.traits.from_literal("datetime'1976-11-23T03:33:06'") assert testdate.year == 1976 assert testdate.second == 6 assert testdate.microsecond == 0 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing without seconds and miliseconds - testdate = typ.traits.from_literal("datetime'1976-11-23T03:33'") + testdate = type_date_time.traits.from_literal("datetime'1976-11-23T03:33'") assert testdate.year == 1976 assert testdate.minute == 33 assert testdate.second == 0 assert testdate.microsecond == 0 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing invalid value with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_literal('xyz') + type_date_time.traits.from_literal('xyz') assert str(e_info.value).startswith('Malformed value xyz for primitive') with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_literal("datetime'xyz'") + type_date_time.traits.from_literal("datetime'xyz'") assert str(e_info.value).startswith('Cannot decode datetime from value xyz') # 3. direction OData -> python # parsing full representation - testdate = typ.traits.from_json("/Date(217567986010)/") + testdate = type_date_time.traits.from_json("/Date(217567986010)/") assert testdate.year == 1976 assert testdate.month == 11 assert testdate.day == 23 @@ -540,38 +566,38 @@ def test_traits_datetime(): assert testdate.minute == 33 assert testdate.second == 6 assert testdate.microsecond == 10000 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing without miliseconds - testdate = typ.traits.from_json("/Date(217567986000)/") + testdate = type_date_time.traits.from_json("/Date(217567986000)/") assert testdate.year == 1976 assert testdate.second == 6 assert testdate.microsecond == 0 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing without seconds and miliseconds - testdate = typ.traits.from_json("/Date(217567980000)/") + testdate = type_date_time.traits.from_json("/Date(217567980000)/") assert testdate.year == 1976 assert testdate.minute == 33 assert testdate.second == 0 assert testdate.microsecond == 0 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing below lowest value with workaround pyodata.v2.model.FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE = True - testdate = typ.traits.from_json("/Date(-62135596800001)/") + testdate = type_date_time.traits.from_json("/Date(-62135596800001)/") assert testdate.year == 1 assert testdate.month == 1 assert testdate.day == 1 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing the lowest value pyodata.v2.model.FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE = False with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_json("/Date(-62135596800001)/") + type_date_time.traits.from_json("/Date(-62135596800001)/") assert str(e_info.value).startswith('Cannot decode datetime from value -62135596800001.') - testdate = typ.traits.from_json("/Date(-62135596800000)/") + testdate = type_date_time.traits.from_json("/Date(-62135596800000)/") assert testdate.year == 1 assert testdate.month == 1 assert testdate.day == 1 @@ -579,23 +605,23 @@ def test_traits_datetime(): assert testdate.minute == 0 assert testdate.second == 0 assert testdate.microsecond == 0 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing above highest value with workaround pyodata.v2.model.FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE = True - testdate = typ.traits.from_json("/Date(253402300800000)/") + testdate = type_date_time.traits.from_json("/Date(253402300800000)/") assert testdate.year == 9999 assert testdate.month == 12 assert testdate.day == 31 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing the highest value pyodata.v2.model.FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE = False with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_json("/Date(253402300800000)/") + type_date_time.traits.from_json("/Date(253402300800000)/") assert str(e_info.value).startswith('Cannot decode datetime from value 253402300800000.') - testdate = typ.traits.from_json("/Date(253402300799999)/") + testdate = type_date_time.traits.from_json("/Date(253402300799999)/") assert testdate.year == 9999 assert testdate.month == 12 assert testdate.day == 31 @@ -603,16 +629,188 @@ def test_traits_datetime(): assert testdate.minute == 59 assert testdate.second == 59 assert testdate.microsecond == 999000 - assert testdate.tzinfo == current_timezone() + assert testdate.tzinfo == timezone.utc # parsing invalid value with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_json("xyz") + type_date_time.traits.from_json("xyz") assert str(e_info.value).startswith('Malformed value xyz for primitive') with pytest.raises(PyODataModelError) as e_info: - typ.traits.from_json("/Date(xyz)/") - assert str(e_info.value).startswith('Cannot decode datetime from value xyz') + type_date_time.traits.from_json("/Date(xyz)/") + assert str(e_info.value).startswith('Malformed value /Date(xyz)/ for primitive Edm.DateTime type.') + + +def test_traits_datetime_with_offset_from_json(type_date_time): + """Test Edm.DateTime with offset""" + + # +10 hours offset, yet must be converted to UTC + testdate = type_date_time.traits.from_json("/Date(217567986010+0600)/") + assert testdate.year == 1976 + assert testdate.month == 11 + assert testdate.day == 23 + assert testdate.hour == 13 # 3 + 10 hours offset + assert testdate.minute == 33 + assert testdate.second == 6 + assert testdate.microsecond == 10000 + assert testdate.tzinfo == timezone.utc + + +@pytest.mark.parametrize('python_datetime,expected,comment', [ + (datetime(1976, 11, 23, 3, 33, 6, microsecond=123000, tzinfo=timezone.utc), '/Date(217567986123)/', 'With milliseconds'), + (datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone.utc), '/Date(217567986000)/', 'No milliseconds'), + ]) +def test_traits_datetime_with_offset_to_json(type_date_time, python_datetime, expected, comment): + """Test Edm.DateTimeOffset trait: Python -> json""" + + assert type_date_time.traits.to_json(python_datetime) == expected, comment + + +def test_traits_datetimeoffset(type_date_time_offset): + """Test Edm.DateTimeOffset traits""" + + assert repr(type_date_time_offset.traits) == 'EdmDateTimeOffsetTypTraits' + + +def test_traits_datetimeoffset_to_literal(type_date_time_offset): + """Test Edm.DateTimeOffset trait: Python -> literal""" + + testdate = datetime(2005, 1, 28, 18, 30, 44, 123456, tzinfo=timezone(timedelta(hours=3, minutes=40))) + assert type_date_time_offset.traits.to_literal(testdate) == "datetimeoffset'2005-01-28T18:30:44.123456+03:40'" + + # without milliseconds part, negative offset + testdate = datetime(2005, 1, 28, 18, 30, 44, 0, tzinfo=timezone(-timedelta(minutes=100))) + assert type_date_time_offset.traits.to_literal(testdate) == "datetimeoffset'2005-01-28T18:30:44-01:40'" + + +def test_traits_invalid_datetimeoffset_to_literal(type_date_time_offset): + # serialization of invalid value + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.to_literal('xyz') + assert str(e_info.value).startswith('Cannot convert value of type') + + +@pytest.mark.parametrize('python_datetime,expected,comment', [ + (datetime(1976, 11, 23, 3, 33, 6, microsecond=123000, tzinfo=timezone.utc), '/Date(217567986123+0000)/', 'UTC'), + (datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=14))), '/Date(217567986000+0840)/', '+14 hours'), + (datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=-12))), '/Date(217567986000-0720)/', '-12 hours'), + ]) +def test_traits_datetimeoffset_to_json(type_date_time_offset, python_datetime, expected, comment): + """Test Edm.DateTimeOffset trait: Python -> json""" + + assert type_date_time_offset.traits.to_json(python_datetime) == expected, comment + + +@pytest.mark.parametrize('literal,expected,comment', [ + ("datetimeoffset'1976-11-23T03:33:06.654321+12:11'", + datetime(1976, 11, 23, 3, 33, 6, microsecond=654321, tzinfo=timezone(timedelta(hours=12, minutes=11))), + 'Full representation'), + ("datetimeoffset'1976-11-23T03:33:06+12:11'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=12, minutes=11))), 'No milliseconds'), + ("datetimeoffset'1976-11-23T03:33:06-01:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=-1))), 'Negative offset'), + ("datetimeoffset'1976-11-23t03:33:06-01:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=-1))), "lowercase 'T' is valid"), + ("datetimeoffset'1976-11-23T03:33:06+00:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone.utc), '+00:00 is UTC'), + ("datetimeoffset'1976-11-23T03:33:06-00:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone.utc), '-00:00 is UTC'), + ("datetimeoffset'1976-11-23t03:33:06Z'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone.utc), 'Z is UTC'), + ("datetimeoffset'1976-11-23t03:33:06+12:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=12))), 'On dateline'), + ("datetimeoffset'1976-11-23t03:33:06-12:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=-12))), 'Minimum offset'), + ("datetimeoffset'1976-11-23t03:33:06+14:00'", datetime(1976, 11, 23, 3, 33, 6, tzinfo=timezone(timedelta(hours=14))), 'Maximum offset'), +]) +def test_traits_datetimeoffset_from_literal(type_date_time_offset, literal, expected, comment): + """Test Edm.DateTimeOffset trait: literal -> Python""" + + assert expected == type_date_time_offset.traits.from_literal(literal), comment + + +def test_traits_datetimeoffset_from_invalid_literal(type_date_time_offset): + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_literal('xyz') + assert str(e_info.value).startswith('Malformed value xyz for primitive') + + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_literal("datetimeoffset'xyz'") + assert str(e_info.value).startswith('Cannot decode datetimeoffset from value xyz') + + +def test_traits_datetimeoffset_from_odata(type_date_time_offset): + """Test Edm.DateTimeOffset trait: OData -> Python""" + + # parsing full representation + testdate = type_date_time_offset.traits.from_json("/Date(217567986010+0060)/") + assert testdate.year == 1976 + assert testdate.month == 11 + assert testdate.day == 23 + assert testdate.hour == 3 + assert testdate.minute == 33 + assert testdate.second == 6 + assert testdate.microsecond == 10000 + assert testdate.tzinfo == timezone(timedelta(hours=1)) + + # parsing without milliseconds, negative offset + testdate = type_date_time_offset.traits.from_json("/Date(217567986000-0005)/") + assert testdate.year == 1976 + assert testdate.minute == 33 + assert testdate.second == 6 + assert testdate.microsecond == 0 + assert testdate.tzinfo == timezone(-timedelta(minutes=5)) + + # parsing below lowest value with workaround + pyodata.v2.model.FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE = True + testdate = type_date_time_offset.traits.from_json("/Date(-62135596800001+0001)/") + assert testdate.year == 1 + assert testdate.month == 1 + assert testdate.day == 1 + assert testdate.minute == 0 + assert testdate.tzinfo == timezone(timedelta(minutes=1)) + + # parsing the lowest value + pyodata.v2.model.FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE = False + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_json("/Date(-62135596800001+0001)/") + assert str(e_info.value).startswith('Cannot decode datetime from value -62135596800001.') + + testdate = type_date_time_offset.traits.from_json("/Date(-62135596800000+0055)/") + assert testdate.year == 1 + assert testdate.month == 1 + assert testdate.day == 1 + assert testdate.hour == 0 + assert testdate.minute == 0 + assert testdate.second == 0 + assert testdate.microsecond == 0 + assert testdate.tzinfo == timezone(timedelta(minutes=55)) + + # parsing above highest value with workaround + pyodata.v2.model.FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE = True + testdate = type_date_time_offset.traits.from_json("/Date(253402300800000+0055)/") + assert testdate.year == 9999 + assert testdate.month == 12 + assert testdate.day == 31 + assert testdate.minute == 0 + assert testdate.tzinfo == timezone(timedelta(minutes=55)) + + # parsing the highest value + pyodata.v2.model.FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE = False + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_json("/Date(253402300800000+0055)/") + assert str(e_info.value).startswith('Cannot decode datetime from value 253402300800000.') + + testdate = type_date_time_offset.traits.from_json("/Date(253402300799999-0001)/") + assert testdate.year == 9999 + assert testdate.month == 12 + assert testdate.day == 31 + assert testdate.hour == 23 + assert testdate.minute == 59 + assert testdate.second == 59 + assert testdate.microsecond == 999000 + assert testdate.tzinfo == timezone(-timedelta(minutes=1)) + + # parsing invalid value + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_json("xyz") + assert str(e_info.value).startswith('Malformed value xyz for primitive') + + with pytest.raises(PyODataModelError) as e_info: + type_date_time_offset.traits.from_json("/Date(xyz)/") + assert str(e_info.value).startswith('Malformed value /Date(xyz)/ for primitive Edm.DateTimeOffset type.') def test_traits_collections(): diff --git a/tests/test_service_v2.py b/tests/test_service_v2.py index 5b69866a..191a0316 100644 --- a/tests/test_service_v2.py +++ b/tests/test_service_v2.py @@ -8,7 +8,7 @@ import pyodata.v2.model import pyodata.v2.service -from pyodata.exceptions import PyODataException, HttpError, ExpressionError, ProgramError +from pyodata.exceptions import PyODataException, HttpError, ExpressionError, ProgramError, PyODataModelError from pyodata.v2.service import EntityKey, EntityProxy, GetEntitySetFilter, ODataHttpResponse, HTTP_CODE_OK from tests.conftest import assert_request_contains_header, contents_of_fixtures_file @@ -286,7 +286,7 @@ def test_entity_key_complex(service): entity_key = { 'Sensor': 'sensor1', - 'Date': datetime.datetime(2017, 12, 24, 18, 0) + 'Date': datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc) } key_properties = set(entity_key.keys()) @@ -345,7 +345,7 @@ def test_entity_key_complex_valid(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), - Sensor='sensor1', Date=datetime.datetime(2017, 12, 24, 18, 0)) + Sensor='sensor1', Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) assert key.to_key_string() == "(Sensor='sensor1',Date=datetime'2017-12-24T18:00:00')" @@ -616,13 +616,13 @@ def test_update_entity(service): request = service.entity_sets.TemperatureMeasurements.update_entity( Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) assert isinstance(request, pyodata.v2.service.EntityModifyRequest) request.set(Value=34.0) # Tests if update entity correctly calls 'to_json' method - request.set(Date=datetime.datetime(2017, 12, 24, 19, 0)) + request.set(Date=datetime.datetime(2017, 12, 24, 19, 0, tzinfo=datetime.timezone.utc)) assert request._values['Value'] == '3.400000E+01' assert request._values['Date'] == '/Date(1514142000000)/' @@ -684,7 +684,7 @@ def test_update_entity_with_entity_key(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key) assert query.get_path() == "TemperatureMeasurements(Sensor='sensor1',Date=datetime'2017-12-24T18:00:00')" @@ -699,7 +699,7 @@ def test_update_entity_with_put_method_specified(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key, method="PUT") assert query.get_method() == "PUT" @@ -714,7 +714,7 @@ def test_update_entity_with_patch_method_specified(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key, method="PATCH") assert query.get_method() == "PATCH" @@ -728,7 +728,7 @@ def test_update_entity_with_merge_method_specified(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key, method='merge') assert query.get_method() == 'MERGE' @@ -743,7 +743,7 @@ def test_update_entity_with_no_method_specified(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key) assert query.get_method() == "PATCH" @@ -758,7 +758,7 @@ def test_update_entity_with_service_config_set_to_put(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) service.config['http']['update_method'] = "PUT" query = service.entity_sets.TemperatureMeasurements.update_entity(key) @@ -774,7 +774,7 @@ def test_update_entity_with_wrong_method_specified(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) with pytest.raises(ValueError) as caught_ex: service.entity_sets.TemperatureMeasurements.update_entity(key, method='DELETE') @@ -790,7 +790,7 @@ def test_get_entity_with_entity_key_and_other_params(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key=key, Foo='Bar') assert query.get_path() == "TemperatureMeasurements(Sensor='sensor1',Date=datetime'2017-12-24T18:00:00')" @@ -807,7 +807,7 @@ def test_get_entity_with_custom_headers(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.get_entity(key) query.add_headers({"X-Foo": "bar"}) @@ -819,7 +819,7 @@ def test_update_entities_with_custom_headers(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.update_entity(key) query.add_headers({"X-Foo": "bar"}) @@ -1380,7 +1380,7 @@ def test_batch_request(service): temp_request = service.entity_sets.TemperatureMeasurements.update_entity( Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)).set(Value=34.0) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)).set(Value=34.0) batch.add_request(employee_request) @@ -1480,7 +1480,7 @@ def test_get_entity_with_entity_key(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.get_entity(key) assert query.get_path() == "TemperatureMeasurements(Sensor='sensor1',Date=datetime'2017-12-24T18:00:00')" @@ -1494,7 +1494,7 @@ def test_get_entity_with_entity_key_and_other_params(service): key = EntityKey( service.schema.entity_type('TemperatureMeasurement'), Sensor='sensor1', - Date=datetime.datetime(2017, 12, 24, 18, 0)) + Date=datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc)) query = service.entity_sets.TemperatureMeasurements.get_entity(key=key, Foo='Bar') assert query.get_path() == "TemperatureMeasurements(Sensor='sensor1',Date=datetime'2017-12-24T18:00:00')" @@ -2259,9 +2259,38 @@ def test_count_with_chained_filters(service): @responses.activate -def test_create_entity_with_datetime(service): +def test_create_entity_with_utc_datetime(service): + """Basic test on creating entity with an UTC datetime object""" + + # pylint: disable=redefined-outer-name + + responses.add( + responses.POST, + f"{service.url}/TemperatureMeasurements", + headers={'Content-type': 'application/json'}, + json={'d': { + 'Sensor': 'Sensor1', + 'Date': '/Date(1514138400000)/', + 'Value': '34.0d' + }}, + status=201) + + request = service.entity_sets.TemperatureMeasurements.create_entity().set(**{ + 'Sensor': 'Sensor1', + 'Date': datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc), + 'Value': 34.0 + }) + + assert request._values['Date'] == '/Date(1514138400000)/' + + result = request.execute() + assert result.Date == datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc) + + +@responses.activate +def test_create_entity_with_non_utc_datetime(service): """ - Basic test on creating entity with datetime + Basic test on creating entity with an non-UTC datetime object Also tzinfo is set to simulate user passing datetime object with different timezone than UTC """ @@ -2294,18 +2323,26 @@ def dst(self, dt): }}, status=201) + with pytest.raises(PyODataModelError) as e_info: + # Offset -18000 sec is for America/Chicago (CDT) timezone + service.entity_sets.TemperatureMeasurements.create_entity().set(**{ + 'Sensor': 'Sensor1', + 'Date': datetime.datetime(2017, 12, 24, 18, 0, tzinfo=MyUTCOffsetTimezone(-18000)), + 'Value': 34.0 + }) - # Offset -18000 sec is for America/Chicago (CDT) timezone - request = service.entity_sets.TemperatureMeasurements.create_entity().set(**{ - 'Sensor': 'Sensor1', - 'Date': datetime.datetime(2017, 12, 24, 18, 0, tzinfo=MyUTCOffsetTimezone(-18000)), - 'Value': 34.0 - }) - assert request._values['Date'] == '/Date(1514138400000)/' +@responses.activate +def test_create_entity_with_naive_datetime(service): + """Preventing creation/usage of an entity with an unaware datetime object""" - result = request.execute() - assert result.Date == datetime.datetime(2017, 12, 24, 18, 0, tzinfo=datetime.timezone.utc) + with pytest.raises(PyODataModelError) as e_info: + service.entity_sets.TemperatureMeasurements.create_entity().set(**{ + 'Sensor': 'Sensor1', + 'Date': datetime.datetime(2017, 12, 24, 18, 0), + 'Value': 34.0 + }) + assert str(e_info.value).startswith('Emd.DateTime accepts only UTC') @responses.activate @@ -2327,7 +2364,7 @@ def test_parsing_of_datetime_before_unix_time(service): request = service.entity_sets.TemperatureMeasurements.create_entity().set(**{ 'Sensor': 'Sensor1', - 'Date': datetime.datetime(1945, 5, 8, 19, 0), + 'Date': datetime.datetime(1945, 5, 8, 19, 0, tzinfo=datetime.timezone.utc), 'Value': 34.0 }) @@ -2337,6 +2374,38 @@ def test_parsing_of_datetime_before_unix_time(service): assert result.Date == datetime.datetime(1945, 5, 8, 19, 0, tzinfo=datetime.timezone.utc) +@responses.activate +@pytest.mark.parametrize("json_input,expected", [ + ('/Date(981173106000+0001)/', datetime.datetime(2001, 2, 3, 4, 5, 6, + tzinfo=datetime.timezone(datetime.timedelta(minutes=1)))), + ('/Date(981173106000-0001)/', datetime.datetime(2001, 2, 3, 4, 5, 6, + tzinfo=datetime.timezone(-datetime.timedelta(minutes=1))))]) +def test_parsing_of_datetimeoffset(service, json_input, expected): + """Test DateTimeOffset handling.""" + + # pylint: disable=redefined-outer-name + + responses.add( + responses.GET, + f"{service.url}/TemperatureMeasurements", + headers={'Content-type': 'application/json'}, + json={'d': { + 'results': [ + { + 'Sensor': 'Sensor1', + 'Date': '/Date(-981173106000)/', + 'DateTimeWithOffset': json_input, + 'Value': '34.0d' + } + ] + }}, + status=200) + + result = service.entity_sets.TemperatureMeasurements.get_entities().execute() + + assert result[0].DateTimeWithOffset == expected + + @responses.activate def test_mismatched_etags_in_body_and_header(service): """Test creating entity with missmatched etags"""