diff --git a/bigtable/docs/snippets.py b/bigtable/docs/snippets.py new file mode 100644 index 000000000000..01564fca50b5 --- /dev/null +++ b/bigtable/docs/snippets.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python + +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Testable usage examples for Google Cloud Bigtable API wrapper + +Each example function takes a ``client`` argument (which must be an instance +of :class:`google.cloud.bigtable.client.Client`) and uses it to perform a task +with the API. + +To facilitate running the examples as system tests, each example is also passed +a ``to_delete`` list; the function adds to the list any objects created which +need to be deleted during teardown. + +.. note:: + This file is under progress and will be updated with more guidance from + the team. Unit tests will be added with guidance from the team. + +""" + +import datetime +import pytest + +from test_utils.system import unique_resource_id +from google.cloud._helpers import UTC +from google.cloud.bigtable import Client +from google.cloud.bigtable import enums + + +INSTANCE_ID = "snippet-" + unique_resource_id('-') +CLUSTER_ID = "clus-1-" + unique_resource_id('-') +LOCATION_ID = 'us-central1-f' +ALT_LOCATION_ID = 'us-central1-a' +PRODUCTION = enums.Instance.Type.PRODUCTION +SERVER_NODES = 3 +STORAGE_TYPE = enums.StorageType.SSD +LABEL_KEY = u'python-snippet' +LABEL_STAMP = datetime.datetime.utcnow() \ + .replace(microsecond=0, tzinfo=UTC,) \ + .strftime("%Y-%m-%dt%H-%M-%S") +LABELS = {LABEL_KEY: str(LABEL_STAMP)} + + +class Config(object): + """Run-time configuration to be modified at set-up. + + This is a mutable stand-in to allow test set-up to modify + global state. + """ + INSTANCE = None + + +def setup_module(): + client = Client(admin=True) + Config.INSTANCE = client.instance(INSTANCE_ID, + instance_type=PRODUCTION, + labels=LABELS) + cluster = Config.INSTANCE.cluster(CLUSTER_ID, + location_id=LOCATION_ID, + serve_nodes=SERVER_NODES, + default_storage_type=STORAGE_TYPE) + operation = Config.INSTANCE.create(clusters=[cluster]) + # We want to make sure the operation completes. + operation.result(timeout=100) + + +def teardown_module(): + Config.INSTANCE.delete() + + +def test_bigtable_create_instance(): + # [START bigtable_create_prod_instance] + from google.cloud.bigtable import Client + from google.cloud.bigtable import enums + + my_instance_id = "inst-my-" + unique_resource_id('-') + my_cluster_id = "clus-my-" + unique_resource_id('-') + location_id = 'us-central1-f' + serve_nodes = 3 + storage_type = enums.StorageType.SSD + production = enums.Instance.Type.PRODUCTION + labels = {'prod-label': 'prod-label'} + + client = Client(admin=True) + instance = client.instance(my_instance_id, instance_type=production, + labels=labels) + cluster = instance.cluster(my_cluster_id, location_id=location_id, + serve_nodes=serve_nodes, + default_storage_type=storage_type) + operation = instance.create(clusters=[cluster]) + # We want to make sure the operation completes. + operation.result(timeout=100) + # [END bigtable_create_prod_instance] + assert instance.exists() + instance.delete() + + +def test_bigtable_create_additional_cluster(): + # [START bigtable_create_cluster] + from google.cloud.bigtable import Client + from google.cloud.bigtable import enums + + # Assuming that there is an existing instance with `INSTANCE_ID` + # on the server already. + # to create an instance see + # 'https://cloud.google.com/bigtable/docs/creating-instance' + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + + cluster_id = "clus-my-" + unique_resource_id('-') + location_id = 'us-central1-a' + serve_nodes = 3 + storage_type = enums.StorageType.SSD + + cluster = instance.cluster(cluster_id, location_id=location_id, + serve_nodes=serve_nodes, + default_storage_type=storage_type) + operation = cluster.create() + # We want to make sure the operation completes. + operation.result(timeout=100) + # [END bigtable_create_cluster] + assert cluster.exists() + + cluster.delete() + + +def test_bigtable_create_app_profile(): + # [START bigtable_create_app_profile] + from google.cloud.bigtable import Client + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + + app_profile_id = "app-prof-" + unique_resource_id('-') + description = 'routing policy-multy' + routing_policy_type = enums.RoutingPolicyType.ANY + + app_profile = instance.app_profile( + app_profile_id=app_profile_id, + routing_policy_type=routing_policy_type, + description=description, + cluster_id=CLUSTER_ID) + + app_profile = app_profile.create(ignore_warnings=True) + # [END bigtable_create_app_profile] + assert app_profile.exists() + + app_profile.delete(ignore_warnings=True) + + +def test_bigtable_list_instances(): + # [START bigtable_list_instances] + from google.cloud.bigtable import Client + + client = Client(admin=True) + (instances_list, failed_locations_list) = client.list_instances() + # [END bigtable_list_instances] + assert len(instances_list) is not 0 + + +def test_bigtable_list_clusters_on_instance(): + # [START bigtable_list_clusters_on_instance] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + (clusters_list, failed_locations_list) = instance.list_clusters() + # [END bigtable_list_clusters_on_instance] + assert len(clusters_list) is not 0 + + +def test_bigtable_list_clusters_in_project(): + # [START bigtable_list_clusters_in_project] + from google.cloud.bigtable import Client + + client = Client(admin=True) + (clusters_list, failed_locations_list) = client.list_clusters() + # [END bigtable_list_clusters_in_project] + assert len(clusters_list) is not 0 + + +def test_bigtable_list_app_profiles(): + # [START bigtable_list_app_profiles] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + # [END bigtable_list_app_profiles] + + app_profile = instance.app_profile( + app_profile_id="app-prof-" + unique_resource_id('-'), + routing_policy_type=enums.RoutingPolicyType.ANY) + app_profile = app_profile.create(ignore_warnings=True) + + # [START bigtable_list_app_profiles] + app_profiles_list = instance.list_app_profiles() + # [END bigtable_list_app_profiles] + assert len(app_profiles_list) is not 0 + + +def test_bigtable_instance_exists(): + # [START bigtable_check_instance_exists] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + instance_exists = instance.exists() + # [END bigtable_check_instance_exists] + assert instance_exists + + +def test_bigtable_cluster_exists(): + # [START bigtable_check_cluster_exists] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + cluster = instance.cluster(CLUSTER_ID) + cluster_exists = cluster.exists() + # [END bigtable_check_cluster_exists] + assert cluster_exists + + +def test_bigtable_reload_instance(): + # [START bigtable_reload_instance] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + instance.reload() + # [END bigtable_reload_instance] + assert instance.type_ is PRODUCTION.value + + +def test_bigtable_reload_cluster(): + # [START bigtable_reload_cluster] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + cluster = instance.cluster(CLUSTER_ID) + cluster.reload() + # [END bigtable_reload_cluster] + assert cluster.serve_nodes is SERVER_NODES + + +def test_bigtable_update_instance(): + # [START bigtable_update_instance] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + display_name = "My new instance" + instance.display_name = display_name + instance.update() + # [END bigtable_update_instance] + assert instance.display_name is display_name + + +def test_bigtable_update_cluster(): + # [START bigtable_update_cluster] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + cluster = instance.cluster(CLUSTER_ID) + cluster.serve_nodes = 8 + cluster.update() + # [END bigtable_update_cluster] + assert cluster.serve_nodes is 8 + + +def test_bigtable_create_table(): + # [START bigtable_create_table] + from google.cloud.bigtable import Client + from google.cloud.bigtable import column_family + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + table = instance.table("table_my") + # Define the GC policy to retain only the most recent 2 versions. + max_versions_rule = column_family.MaxVersionsGCRule(2) + table.create(column_families={'cf1': max_versions_rule}) + # [END bigtable_create_table] + assert table.exists() + + +def test_bigtable_list_tables(): + # [START bigtable_list_tables] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + tables_list = instance.list_tables() + # [END bigtable_list_tables] + assert len(tables_list) is not 0 + + +def test_bigtable_delete_cluster(): + # [START bigtable_delete_cluster] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + cluster_id = "clus-my-" + unique_resource_id('-') + # [END bigtable_delete_cluster] + + cluster = instance.cluster(cluster_id, location_id=ALT_LOCATION_ID, + serve_nodes=SERVER_NODES, + default_storage_type=STORAGE_TYPE) + operation = cluster.create() + # We want to make sure the operation completes. + operation.result(timeout=1000) + + # [START bigtable_delete_cluster] + cluster_to_delete = instance.cluster(cluster_id) + cluster_to_delete.delete() + # [END bigtable_delete_cluster] + assert not cluster_to_delete.exists() + + +def test_bigtable_delete_instance(): + # [START bigtable_delete_instance] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance_id_to_delete = "inst-my-" + unique_resource_id('-') + # [END bigtable_delete_instance] + cluster_id = "clus-my-" + unique_resource_id('-') + + instance = client.instance(instance_id_to_delete, + instance_type=PRODUCTION, + labels=LABELS) + cluster = instance.cluster(cluster_id, + location_id=ALT_LOCATION_ID, + serve_nodes=SERVER_NODES, + default_storage_type=STORAGE_TYPE) + operation = instance.create(clusters=[cluster]) + # We want to make sure the operation completes. + operation.result(timeout=100) + + # [START bigtable_delete_instance] + instance_to_delete = client.instance(instance_id_to_delete) + instance_to_delete.delete() + # [END bigtable_delete_instance] + assert not instance_to_delete.exists() + + +def test_bigtable_test_iam_permissions(): + # [START bigtable_test_iam_permissions] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + instance.reload() + permissions = ["bigtable.clusters.create", "bigtable.tables.create"] + permissions_allowed = instance.test_iam_permissions(permissions) + # [END bigtable_test_iam_permissions] + assert permissions_allowed == permissions + + +def test_bigtable_get_iam_policy(): + # [START bigtable_get_iam_policy] + from google.cloud.bigtable import Client + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + policy_latest = instance.get_iam_policy() + # [END bigtable_get_iam_policy] + + assert len(policy_latest.bigtable_viewers) is not 0 + + +def test_bigtable_set_iam_policy(): + # [START bigtable_set_iam_policy] + from google.cloud.bigtable import Client + from google.cloud.bigtable.policy import Policy + from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE + + client = Client(admin=True) + instance = client.instance(INSTANCE_ID) + instance.reload() + ins_policy = Policy() + ins_policy[BIGTABLE_ADMIN_ROLE] = [ + Policy.user("test_iam@test.com"), + Policy.service_account("sv_account@gmail.com")] + + policy_latest = instance.set_iam_policy(ins_policy) + # [END bigtable_set_iam_policy] + + assert len(policy_latest.bigtable_admins) is not 0 + + +if __name__ == '__main__': + pytest.main() diff --git a/bigtable/google/cloud/bigtable/client.py b/bigtable/google/cloud/bigtable/client.py index 1ef9e072199c..76bb3706190a 100644 --- a/bigtable/google/cloud/bigtable/client.py +++ b/bigtable/google/cloud/bigtable/client.py @@ -203,6 +203,12 @@ def instance(self, instance_id, display_name=None, instance_type=None, labels=None): """Factory to create a instance associated with this client. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_prod_instance] + :end-before: [END bigtable_create_prod_instance] + :type instance_id: str :param instance_id: The ID of the instance. @@ -241,6 +247,12 @@ def instance(self, instance_id, display_name=None, def list_instances(self): """List instances owned by the project. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_list_instances] + :end-before: [END bigtable_list_instances] + :rtype: tuple :returns: (instances, failed_locations), where 'instances' is list of @@ -256,6 +268,12 @@ def list_instances(self): def list_clusters(self): """List the clusters in the project. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_list_clusters_in_project] + :end-before: [END bigtable_list_clusters_in_project] + :rtype: tuple :returns: (clusters, failed_locations), where 'clusters' is list of diff --git a/bigtable/google/cloud/bigtable/cluster.py b/bigtable/google/cloud/bigtable/cluster.py index b5032f805f10..1b3fe559c3c7 100644 --- a/bigtable/google/cloud/bigtable/cluster.py +++ b/bigtable/google/cloud/bigtable/cluster.py @@ -172,7 +172,14 @@ def __ne__(self, other): return not self == other def reload(self): - """Reload the metadata for this cluster.""" + """Reload the metadata for this cluster. + + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_reload_cluster] + :end-before: [END bigtable_reload_cluster] + """ cluster_pb = self._instance._client.instance_admin_client.get_cluster( self.name) @@ -183,6 +190,12 @@ def reload(self): def exists(self): """Check whether the cluster already exists. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_check_cluster_exists] + :end-before: [END bigtable_check_cluster_exists] + :rtype: bool :returns: True if the table exists, else False. """ @@ -197,6 +210,12 @@ def exists(self): def create(self): """Create this cluster. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_cluster] + :end-before: [END bigtable_create_cluster] + .. note:: Uses the ``project``, ``instance`` and ``cluster_id`` on the @@ -223,6 +242,12 @@ def create(self): def update(self): """Update this cluster. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_update_cluster] + :end-before: [END bigtable_update_cluster] + .. note:: Updates the ``serve_nodes``. If you'd like to @@ -260,6 +285,12 @@ def update(self): def delete(self): """Delete this cluster. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_delete_cluster] + :end-before: [END bigtable_delete_cluster] + Marks a cluster and all of its tables for permanent deletion in 7 days. Immediately upon completion of the request: diff --git a/bigtable/google/cloud/bigtable/instance.py b/bigtable/google/cloud/bigtable/instance.py index b09470dc60b7..17e373673f64 100644 --- a/bigtable/google/cloud/bigtable/instance.py +++ b/bigtable/google/cloud/bigtable/instance.py @@ -187,7 +187,14 @@ def __ne__(self, other): return not self == other def reload(self): - """Reload the metadata for this instance.""" + """Reload the metadata for this instance. + + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_reload_instance] + :end-before: [END bigtable_reload_instance] + """ instance_pb = self._client.instance_admin_client.get_instance( self.name) @@ -198,6 +205,12 @@ def reload(self): def exists(self): """Check whether the instance already exists. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_check_instance_exists] + :end-before: [END bigtable_check_instance_exists] + :rtype: bool :returns: True if the table exists, else False. """ @@ -213,6 +226,12 @@ def create(self, location_id=None, default_storage_type=None, clusters=None): """Create this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_prod_instance] + :end-before: [END bigtable_create_prod_instance] + .. note:: Uses the ``project`` and ``instance_id`` on the current @@ -287,6 +306,12 @@ def create(self, location_id=None, def update(self): """Updates an instance within a project. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_update_instance] + :end-before: [END bigtable_update_instance] + .. note:: Updates any or all of the following values: @@ -324,6 +349,12 @@ def update(self): def delete(self): """Delete this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_delete_instance] + :end-before: [END bigtable_delete_instance] + Marks an instance and all of its tables for permanent deletion in 7 days. @@ -349,6 +380,12 @@ def cluster(self, cluster_id, location_id=None, serve_nodes=None, default_storage_type=None): """Factory to create a cluster associated with this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_cluster] + :end-before: [END bigtable_create_cluster] + :type cluster_id: str :param cluster_id: The ID of the cluster. @@ -385,6 +422,12 @@ def cluster(self, cluster_id, location_id=None, def list_clusters(self): """List the clusters in this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_list_clusters_on_instance] + :end-before: [END bigtable_list_clusters_on_instance] + :rtype: tuple :returns: (clusters, failed_locations), where 'clusters' is list of @@ -400,6 +443,12 @@ def list_clusters(self): def table(self, table_id, app_profile_id=None): """Factory to create a table associated with this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_table] + :end-before: [END bigtable_create_table] + :type table_id: str :param table_id: The ID of the table. @@ -414,6 +463,12 @@ def table(self, table_id, app_profile_id=None): def list_tables(self): """List the tables in this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_list_tables] + :end-before: [END bigtable_list_tables] + :rtype: list of :class:`Table ` :returns: The list of tables owned by the instance. :raises: :class:`ValueError ` if one of the @@ -439,6 +494,12 @@ def app_profile(self, app_profile_id, allow_transactional_writes=None): """Factory to create AppProfile associated with this instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_create_app_profile] + :end-before: [END bigtable_create_app_profile] + :type app_profile_id: str :param app_profile_id: The ID of the AppProfile. Must be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. @@ -475,6 +536,12 @@ def app_profile(self, app_profile_id, def list_app_profiles(self): """Lists information about AppProfiles in an instance. + For example: + + .. literalinclude:: snippets.py + :start-after: [START bigtable_list_app_profiles] + :end-before: [END bigtable_list_app_profiles] + :rtype: :list:[`~google.cloud.bigtable.app_profile.AppProfile`] :returns: A :list:[`~google.cloud.bigtable.app_profile.AppProfile`]. By default, this is a list of @@ -487,15 +554,11 @@ def list_app_profiles(self): def get_iam_policy(self): """Gets the access control policy for an instance resource. - .. code-block:: python - - from google.cloud.bigtable.client import Client - from google.cloud.bigtable.policy import Policy + For example: - client = Client(admin=True) - instance = client.instance('[INSTANCE_ID]') - policy_latest = instance.get_iam_policy() - print (policy_latest.bigtable_viewers) + .. literalinclude:: snippets.py + :start-after: [START bigtable_get_iam_policy] + :end-before: [END bigtable_get_iam_policy] :rtype: :class:`google.cloud.bigtable.policy.Policy` :returns: The current IAM policy of this instance @@ -511,21 +574,11 @@ def set_iam_policy(self, policy): For more information about policy, please see documentation of class `google.cloud.bigtable.policy.Policy` - .. code-block:: python + For example: - from google.cloud.bigtable.client import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE - - client = Client(admin=True) - instance = client.instance('[INSTANCE_ID]') - ins_policy = instance.get_iam_policy() - ins_policy[BIGTABLE_ADMIN_ROLE] = [ - Policy.user("test_iam@test.com"), - Policy.service_account("sv_account@gmail.com")] - - policy_latest = instance.set_iam_policy() - print (policy_latest.bigtable_admins) + .. literalinclude:: snippets.py + :start-after: [START bigtable_set_iam_policy] + :end-before: [END bigtable_set_iam_policy] :type policy: :class:`google.cloud.bigtable.policy.Policy` :param policy: A new IAM policy to replace the current IAM policy @@ -543,16 +596,11 @@ def test_iam_permissions(self, permissions): """Returns permissions that the caller has on the specified instance resource. - .. code-block:: python - - from google.cloud.bigtable.client import Client + For example: - client = Client(admin=True) - instance = client.instance('[INSTANCE_ID]') - permissions = ["bigtable.tables.create", - "bigtable.clusters.create"] - permissions_allowed = instance.test_iam_permissions(permissions) - print (permissions_allowed) + .. literalinclude:: snippets.py + :start-after: [START bigtable_test_iam_permissions] + :end-before: [END bigtable_test_iam_permissions] :type permissions: list :param permissions: The set of permissions to check for diff --git a/bigtable/noxfile.py b/bigtable/noxfile.py index 28738b0c42fb..dfef10881d03 100644 --- a/bigtable/noxfile.py +++ b/bigtable/noxfile.py @@ -91,7 +91,7 @@ def lint(session): """ session.install('flake8', *LOCAL_DEPS) session.install('.') - session.run('flake8', 'google', 'tests') + session.run('flake8', 'google', 'tests', 'docs') @nox.session(python='3.6') @@ -113,3 +113,22 @@ def cover(session): session.install('coverage', 'pytest-cov') session.run('coverage', 'report', '--show-missing', '--fail-under=100') session.run('coverage', 'erase') + + +@nox.session(python='3.7') +def snippets(session): + """Run the system test suite.""" + # Sanity check: Only run system tests if the environment variable is set. + if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''): + session.skip('Credentials must be set via environment variable.') + + # Install all test dependencies, then install local packages in place. + session.install('mock', 'pytest') + for local_dep in LOCAL_DEPS: + session.install('-e', local_dep) + session.install('-e', os.path.join('..', 'bigtable')) + session.install('-e', '../test_utils/') + session.install('-e', '.') + session.run('py.test', '--quiet', \ + os.path.join('docs', 'snippets.py'), \ + *session.posargs)