Skip to content

Fix tests for get_standard_metadata #134

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
luandy64 merged 11 commits into from
Dec 4, 2020
Merged

Fix tests for get_standard_metadata #134

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
a38845d
Fix testing of get_standard_metadata
judahrand Oct 29, 2020
c163cff
Fix maxDiff issue
Dec 2, 2020
e5b8fb2
Add docstring
Dec 2, 2020
d968992
Revert change to get_standard_metadata()
Dec 2, 2020
adc8a6c
Update how asserts work
Dec 2, 2020
d9e7697
Fix missing import
Dec 2, 2020
21ec743
Revert changes to breadcrumbs
Dec 2, 2020
994a9f6
Rename test_kp to has_pk
Dec 2, 2020
9d5f805
Add test numbers
Dec 2, 2020
1d5c327
Add missing test
Dec 2, 2020
3c16458
Fix failing test
Dec 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 80 additions & 58 deletions tests/test_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from pprint import pprint
import unittest
from singer.metadata import get_standard_metadata
from singer.metadata import get_standard_metadata, to_map

def make_expected_metadata(base_obj, dict_of_extras):
def make_expected_metadata(base_obj, dict_of_extras, has_pk=False):
metadata_value = {**base_obj}
metadata_value.update(dict_of_extras)

Expand All @@ -13,7 +13,7 @@ def make_expected_metadata(base_obj, dict_of_extras):
},
{
'metadata': {
'inclusion': 'available',
'inclusion': 'automatic' if has_pk else 'available',
},
'breadcrumb': ('properties', 'id')
},
Expand All @@ -33,9 +33,19 @@ def make_expected_metadata(base_obj, dict_of_extras):

class TestStandardMetadata(unittest.TestCase):

#maxDiff = None

def test_standard_metadata(self):
"""
There's four inputs we want to test: schema, key_properties, replication_method, valid_replication_keys.

When `schema` is a non-null input, we expect `"inclusion": "available"` metadata for the `()` breadcrumb.

When `key_properties` is a non-null input, we expect `table-key-properties` metadata for the `()` breadcrumb.

When `replication_method` is a non-null input, we expect `forced-replication-method` metadata for the `()` breadcrumb.

When `valid_replication_keys` is a non-null input, we expect `valid-replication-keys` metadata for the `()` breadcrumb.
"""
self.maxDiff = None

# Some contants shared by a number of expected metadata objects
tap_stream_id = 'employees'
Expand All @@ -44,7 +54,7 @@ def test_standard_metadata(self):
test_rk = ['id', 'created']
metadata_kp = {'table-key-properties': ['id']}
metadata_rm = {'forced-replication-method': 'INCREMENTAL'}
metadata_rk = {'valid_replication_keys': ['id','created']}
metadata_rk = {'valid-replication-keys': ['id','created']}
schema_present_base_obj = {'inclusion': 'available'}
test_schema = {
'type': ['null', 'object'],
Expand All @@ -61,7 +71,7 @@ def test_standard_metadata(self):
# dictionary of parameters for `get_standard_metadata()` and the
# second element is the expected metadata
test_variables = [
(
( # test_number=0
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -74,7 +84,7 @@ def test_standard_metadata(self):
{'schema-name': tap_stream_id,}
)
),
(
( # test_number=1
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -84,11 +94,11 @@ def test_standard_metadata(self):
},
make_expected_metadata(
schema_present_base_obj,
{'valid_replication_keys': ['id','created'],
{'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id}
)
),
(
( # test_number=2
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -102,7 +112,7 @@ def test_standard_metadata(self):
'schema-name':tap_stream_id}
)
),
(
( # test_number=3
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -112,12 +122,12 @@ def test_standard_metadata(self):
},
make_expected_metadata(
schema_present_base_obj,
{'valid_replication_keys': ['id','created'],
{'valid-replication-keys': ['id','created'],
'forced-replication-method': 'INCREMENTAL',
'schema-name':tap_stream_id}
)
),
(
( # test_number=4
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -128,10 +138,11 @@ def test_standard_metadata(self):
make_expected_metadata(
schema_present_base_obj,
{'table-key-properties': ['id'],
'schema-name':tap_stream_id}
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=5
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -140,14 +151,14 @@ def test_standard_metadata(self):
'valid_replication_keys': test_rk
},
make_expected_metadata(

schema_present_base_obj,
{'table-key-properties': ['id'],
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id}
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=6
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -159,10 +170,11 @@ def test_standard_metadata(self):
schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'schema-name':tap_stream_id}
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=7
{
'schema': test_schema,
'schema_name': tap_stream_id,
Expand All @@ -174,25 +186,21 @@ def test_standard_metadata(self):
schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id}
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True
)
),
(
( # test_number=8
{
'schema': None,
'key_properties': None,
'replication_method': None,
'valid_replication_keys': None
},
[
{
'metadata': {},
'breadcrumb': []
}
]
[]
),
(
( # test_number=9
{
'schema': None,
'key_properties': None,
Expand All @@ -202,14 +210,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
(
( # test_number=10
{
'schema': None,
'key_properties': None,
Expand All @@ -219,14 +226,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'forced-replication-method': 'INCREMENTAL'
},
'breadcrumb': []
}
]
),
(
( # test_number=11
{
'schema': None,
'key_properties': None,
Expand All @@ -236,15 +242,14 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
(
( # test_number=12
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -254,14 +259,13 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
},
'breadcrumb': []
}
]
),
(
( # test_number=13
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -271,15 +275,31 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
),
( # test_number=14
{
'schema': None,
'key_properties': test_kp,
'replication_method': test_rm,
'valid_replication_keys': None
},
[
{
'metadata': {
'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
},
'breadcrumb': []
}
]
),
(
( # test_number=15
{
'schema': None,
'key_properties': test_kp,
Expand All @@ -289,26 +309,26 @@ def test_standard_metadata(self):
[
{
'metadata': {
'inclusion': 'available',
'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created']
'valid-replication-keys': ['id','created']
},
'breadcrumb': []
}
]
)
]

for var in test_variables:
function_params = var[0]
expected_metadata = var[1]
for i, var in enumerate(test_variables):
with self.subTest(test_number=i):
function_params = var[0]
expected_metadata = var[1]

test_value = get_standard_metadata(**function_params)
test_value = get_standard_metadata(**function_params)

for obj in expected_metadata:
if obj in test_value:
self.assertIn(obj, test_value)
expected_value = to_map(expected_metadata)
actual_value = to_map(test_value)
self.assertDictEqual(expected_value, actual_value)

# Test one function call where the parameters are not splat in
test_value = get_standard_metadata(test_schema,
Expand All @@ -320,11 +340,13 @@ def test_standard_metadata(self):
expected_metadata = make_expected_metadata(schema_present_base_obj,
{'table-key-properties': ['id'],
'forced-replication-method': 'INCREMENTAL',
'valid_replication_keys': ['id','created'],
'schema-name':tap_stream_id})
for obj in expected_metadata:
if obj in test_value:
self.assertIn(obj, test_value)
'valid-replication-keys': ['id','created'],
'schema-name':tap_stream_id},
has_pk=True)
self.assertDictEqual(
to_map(expected_metadata),
to_map(test_value)
)

def test_empty_key_properties_are_written(self):
mdata = get_standard_metadata(key_properties=[])
Expand Down