diff --git a/.ratignore b/.ratignore index 3f0f82e0da..f113a4d195 100644 --- a/.ratignore +++ b/.ratignore @@ -6,13 +6,13 @@ CHANGES HACKING test/storage/fixtures/ test/compute/fixtures/ +test/loadbalancer/fixtures/ coverage_html_report/ .coverage .coveragerc -data/pricing.json -test/pricing_test.json - -common/__init__.py -compute/__init__.py -storage/__init__.py +libcloud/data/pricing.json +libcloud/common/__init__.py +libcloud/compute/__init__.py +libcloud/storage/__init__.py test/storage/__init__.py +test/pricing_test.json diff --git a/CHANGES b/CHANGES index 1e867d01de..f14ba13266 100644 --- a/CHANGES +++ b/CHANGES @@ -1,85 +1,70 @@ -*- coding: utf-8 -*- -Changes with Apache Libcloud 0.4.3 - *) Add implementation for the following methods to the CloudFiles - storage driver: enable_container_cdn(), get_container_cdn_url(), - get_object_cdn_url() +Changes with Apache Libcloud 0.5.1 + *) Rackspace driver: + - Properly handle response errors and only throw InvalidCredsError if + the returned status code is 401 + [Brad Morgan] + + *) Nimbus driver: + - Fix the create_node method and make the "ex_create_tag" method a no-op, + because Nimbus doesn't support creating tags. + +Changes with Apache Libcloud 0.5.0 + + *) Existing APIs directly on the libcloud.* module have been + deprecated and will be removed in version 0.6.0. Most methods + were moved to the libcloud.compute.* module. + + *) Add new libcloud.loadbalancers API, with initial support for: + - GoGrid Load Balancers + - Rackspace Load Balancers + [Roman Bogorodskiy] + + *) Add new libcloud.storage API, with initial support for: + - Amazon S3 + - Rackspace CloudFiles [Tomaz Muraus] - *) Add OpenStack driver, which is actually an extension for - Rackspace driver that allows to specify custom port and - host of user's OpenStack installation - [Roman Bogorodskiy] - - *) Add support for Load Balancing services, available through - libcloud.resource.lb. Drivers for Rackspace and GoGrid are - included. - [Roman Bogorodskiy] - - *) Update GoGrid driver to use API Version 1.8. Sandbox flag - for servers is no longer used. - [Roman Bogorodskiy] - - *) When creating an EC2 node, don't ignore the name argument - and create a "Name" tag with the value of this argument. - [Tomaz Muraus] - - *) Add Gandi.net driver. - [Aymeric Barantal] - - *) Add extension method for modifying node attributes and - changing the node size (EC2 driver). - [Tomaz Muraus] - - *) Add Bluebox driver. - [Christian Paredes] - - *) Add Nimbus driver. - [David LaBissoniere] - - *) Minor fixes to get the library and tests working on - Python 2.7 and PyPy. - [Tomaz Muraus] + *) Add new libcloud.compute drivers for: + - Bluebox [Christian Paredes] + - Gandi.net [Aymeric Barantal] + - Nimbus [David LaBissoniere] + - OpenStack [Roman Bogorodskiy] + - Opsource.net [Joe Miller] *) Added "pricing" module and improved pricing handling. [Tomaz Muraus] - *) Add support for the new Amazon Region (Tokyo) - [Tomaz Muraus] - - *) Implement ex_rebuild() and ex_get_node_details() - routines for Rackspace driver. - [Andrew Klochkov] - - *) Implement ex_list_ips() for GoGrid driver to list - IP addresses assigned to the account. Make use of - it in _get_first_ip(). + *) Updates to the GoGrid compute driver: + - Use API version 1.0. + - Remove sandbox flag. + - Add ex_list_ips() to list IP addresses assigned to the account. + - Implement ex_edit_image method which allows changing image attributes + like name, description and make image public or private. [Roman Bogorodskiy] - *) Added ex_create_tags and ex_delete_tags methods to the - EC2 driver. - [Brandon Rhodes] - - *) Include node Elastic IP addresses in the node public_ip - attribute for the EC2 nodes. - [Tomaz Muraus] - - *) Use ipAddress and privateIpAddress attribute for - the EC 2node public and private ip - [Tomaz Muraus] - - *) Add ex_describe_addresses method to the EC2 driver. + *) Updates to the Amazon EC2 compute driver: + - When creating a Node, use the name argument to set a Tag with the + value. [Tomaz Muraus] + - Add extension method for modifying node attributes and changing the + node size. [Tomaz Muraus] + - Add support for the new Amazon Region (Tokyo). [Tomaz Muraus] + - Added ex_create_tags and ex_delete_tags. [Brandon Rhodes] + - Include node Elastic IP addresses in the node public_ip attribute + for the EC2 nodes. [Tomaz Muraus] + - Use ipAddress and privateIpAddress attribute for the EC 2node public + and private ip. [Tomaz Muraus] + - Add ex_describe_addresses method to the EC2 driver. [Tomaz Muraus] + + *) Updates to the Rackspace CloudServers compute driver: + - Add ex_rebuild() and ex_get_node_details() [Andrew Klochkov] + - Expose URI of a Rackspace node to the node meta data. [Paul Querna] + + *) Minor fixes to get the library and tests working on Python 2.7 and PyPy. [Tomaz Muraus] - *) Expose URI of a Rackspace node to the node meta data. - [Paul Querna] - - *) Implement ex_edit_image method for GoGrid driver - which allows changing image attributes like name, - description and make image public or private. - [Roman Bogorodskiy] - -Changes with Apache Libcloud 0.4.2 +Changes with Apache Libcloud 0.4.2 (Released January 18, 2011) *) Fix EC2 create_node to become backward compatible for NodeLocation. diff --git a/MANIFEST.in b/MANIFEST.in index 40aa1d1604..38d865cac1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,13 +1,20 @@ include LICENSE include NOTICE include DISCLAIMER -include example.py +include example_*.py include CONTRIBUTORS include CHANGES include HACKING include README +include libcloud/data/pricing.json prune test/secrets.py -include test/*.py include demos/* +include test/*.py +include test/pricing_test.json include test/secrets.py-dist -include test/fixtures/*/* \ No newline at end of file +include test/compute/*.py +include test/storage/*.py +include test/loadbalancer/*.py +include test/compute/fixtures/*/* +include test/storage/fixtures/*/* +include test/loadbalancer/fixtures/*/* diff --git a/example_lb.py b/example_lb.py deleted file mode 100644 index b4cc69f3b1..0000000000 --- a/example_lb.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -import os -import time - -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.types import Provider, LBState -from libcloud.resource.lb.providers import get_driver - -def main(): - Rackspace = get_driver(Provider.RACKSPACE) - - driver = Rackspace('username', 'api key') - - balancers = driver.list_balancers() - - # creating a balancer which balances traffic across two - # nodes: 192.168.86.1:80 and 192.168.86.2:8080. Balancer - # itself listens on port 80/tcp - new_balancer_name = 'testlb' + os.urandom(4).encode('hex') - new_balancer = driver.create_balancer(name=new_balancer_name, - port=80, - nodes=(LBNode(None, '192.168.86.1', 80), - LBNode(None, '192.168.86.2', 8080)) - ) - - print new_balancer - - # wait for balancer to become ready - # NOTE: in real life code add timeout to not end up in - # endless loop when things go wrong on provider side - while True: - balancer = driver.balancer_detail(balancer=new_balancer) - - if balancer.state == LBState.RUNNING: - break - - time.sleep(30) - - # fetch list of nodes - nodes = balancer.list_nodes() - print nodes - - # remove first node - balancer.detach_node(nodes[0]) - - # and add another one: 10.0.0.10:1000 - print balancer.attach_node(ip='10.0.0.10', port='1000') - - # remove the balancer - driver.destroy_balancer(new_balancer) - -if __name__ == "__main__": - main() diff --git a/example_loadbalancer.py b/example_loadbalancer.py new file mode 100644 index 0000000000..61ad5ac22c --- /dev/null +++ b/example_loadbalancer.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import time + +from libcloud.loadbalancer.base import Member, Algorithm +from libcloud.loadbalancer.types import Provider, State +from libcloud.loadbalancer.providers import get_driver + +def main(): + Rackspace = get_driver(Provider.RACKSPACE_US) + + driver = Rackspace('username', 'api key') + + balancers = driver.list_balancers() + + print balancers + + # creating a balancer which balances traffic across two + # nodes: 192.168.86.1:80 and 192.168.86.2:8080. Balancer + # itself listens on port 80/tcp + new_balancer_name = 'testlb' + os.urandom(4).encode('hex') + new_balancer = driver.create_balancer(name=new_balancer_name, + algorithm=Algorithm.ROUND_ROBIN, + port=80, + protocol='http', + members=(Member(None, '192.168.86.1', 80), + Member(None, '192.168.86.2', 8080)) + ) + + print new_balancer + + # wait for balancer to become ready + # NOTE: in real life code add timeout to not end up in + # endless loop when things go wrong on provider side + while True: + balancer = driver.get_balancer(balancer_id=new_balancer.id) + + if balancer.state == State.RUNNING: + break + + print "sleeping for 30 seconds for balancers to become ready" + time.sleep(30) + + # fetch list of members + members = balancer.list_members() + print members + + # remove first member + balancer.detach_member(members[0]) + + # remove the balancer + driver.destroy_balancer(new_balancer) + +if __name__ == "__main__": + main() diff --git a/libcloud/__init__.py b/libcloud/__init__.py index a6a5eaced7..68c57195a7 100644 --- a/libcloud/__init__.py +++ b/libcloud/__init__.py @@ -21,7 +21,7 @@ __all__ = ["__version__", "enable_debug"] -__version__ = "0.5.0-dev" +__version__ = "0.5.0" def enable_debug(fo): """ diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 0932de9aa2..896c87e90b 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -25,39 +25,6 @@ from libcloud.httplib_ssl import LibcloudHTTPSConnection from httplib import HTTPConnection as LibcloudHTTPConnection -class RawResponse(object): - - def __init__(self, response=None): - self._status = None - self._response = None - self._headers = {} - self._error = None - self._reason = None - - @property - def response(self): - if not self._response: - self._response = self.connection.connection.getresponse() - return self._response - - @property - def status(self): - if not self._status: - self._status = self.response.status - return self._status - - @property - def headers(self): - if not self._headers: - self._headers = dict(self.response.getheaders()) - return self._headers - - @property - def reason(self): - if not self._reason: - self._reason = self.response.reason - return self._reason - class Response(object): """ A Base Response class to derive from. @@ -113,6 +80,43 @@ def success(self): """ return self.status == httplib.OK or self.status == httplib.CREATED +class RawResponse(Response): + + def __init__(self, response=None): + self._status = None + self._response = None + self._headers = {} + self._error = None + self._reason = None + + @property + def response(self): + if not self._response: + response = self.connection.connection.getresponse() + self._response, self.body = response, response + if not self.success(): + self.parse_error() + return self._response + + @property + def status(self): + if not self._status: + self._status = self.response.status + return self._status + + @property + def headers(self): + if not self._headers: + self._headers = dict(self.response.getheaders()) + return self._headers + + @property + def reason(self): + if not self._reason: + self._reason = self.response.reason + return self._reason + + #TODO: Move this to a better location/package class LoggingConnection(): """ @@ -375,7 +379,7 @@ def request(self, # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: - self.connection.putrequest(method, action) + self.connection.putrequest(method, url) for key, value in headers.iteritems(): self.connection.putheader(key, value) diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py new file mode 100644 index 0000000000..c3d5b0194a --- /dev/null +++ b/libcloud/common/openstack.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Common utilities for Rackspace Cloud Servers and Cloud Files +""" +import httplib +from urllib2 import urlparse +from libcloud.common.base import ConnectionUserAndKey +from libcloud.compute.types import InvalidCredsError, MalformedResponseError + +AUTH_API_VERSION = 'v1.0' + +__all__ = [ + "OpenstackBaseConnection", + ] + +class OpenstackBaseConnection(ConnectionUserAndKey): + def __init__(self, user_id, key, secure): + self.cdn_management_url = None + self.storage_url = None + self.auth_token = None + self.__host = None + super(OpenstackBaseConnection, self).__init__( + user_id, key, secure=secure) + + def add_default_headers(self, headers): + headers['X-Auth-Token'] = self.auth_token + headers['Accept'] = self.accept_format + return headers + + @property + def request_path(self): + return self._get_request_path(url_key=self._url_key) + + @property + def host(self): + # Default to server_host + return self._get_host(url_key=self._url_key) + + def _get_request_path(self, url_key): + value_key = '__request_path_%s' % (url_key) + value = getattr(self, value_key, None) + + if not value: + self._populate_hosts_and_request_paths() + value = getattr(self, value_key, None) + + return value + + def _get_host(self, url_key): + value_key = '__%s' % (url_key) + value = getattr(self, value_key, None) + + if not value: + self._populate_hosts_and_request_paths() + value = getattr(self, value_key, None) + + return value + + def _populate_hosts_and_request_paths(self): + """ + Rackspace uses a separate host for API calls which is only provided + after an initial authentication request. If we haven't made that + request yet, do it here. Otherwise, just return the management host. + """ + if not self.auth_token: + # Initial connection used for authentication + conn = self.conn_classes[self.secure]( + self.auth_host, self.port[self.secure]) + conn.request( + method='GET', + url='/%s' % (AUTH_API_VERSION), + headers={ + 'X-Auth-User': self.user_id, + 'X-Auth-Key': self.key + } + ) + + resp = conn.getresponse() + + if resp.status == httplib.NO_CONTENT: + # HTTP NO CONTENT (204): auth successful + headers = dict(resp.getheaders()) + + try: + self.storage_url = headers['x-storage-url'] + self.auth_token = headers['x-auth-token'] + except KeyError, e: + # Returned 204 but has missing information in the header, something is wrong + raise MalformedResponseError('Malformed response', + body='Missing header: %s' % (str(e)), + driver=self.driver) + elif resp.status == httplib.UNAUTHORIZED: + # HTTP UNAUTHORIZED (401): auth failed + raise InvalidCredsError() + else: + # Any response code != 401 or 204, something is wrong + raise MalformedResponseError('Malformed response', + body='code: %s body:%s' % (resp.status, ''.join(resp.body.readlines())), + driver=self.driver) + + for key in ['storage_url']: + scheme, server, request_path, param, query, fragment = ( + urlparse.urlparse(getattr(self, key))) + # Set host to where we want to make further requests to + setattr(self, '__%s' % (key), server) + setattr(self, '__request_path_%s' % (key), request_path) + + conn.close() diff --git a/libcloud/common/rackspace.py b/libcloud/common/rackspace.py index 953a96147b..f176012cab 100644 --- a/libcloud/common/rackspace.py +++ b/libcloud/common/rackspace.py @@ -19,7 +19,7 @@ import httplib from urllib2 import urlparse from libcloud.common.base import ConnectionUserAndKey -from libcloud.compute.types import InvalidCredsError +from libcloud.compute.types import InvalidCredsError, MalformedResponseError AUTH_HOST_US='auth.api.rackspacecloud.com' AUTH_HOST_UK='lon.auth.api.rackspacecloud.com' @@ -95,19 +95,29 @@ def _populate_hosts_and_request_paths(self): resp = conn.getresponse() - if resp.status != httplib.NO_CONTENT: - raise InvalidCredsError() - - headers = dict(resp.getheaders()) - - try: - self.server_url = headers['x-server-management-url'] - self.storage_url = headers['x-storage-url'] - self.cdn_management_url = headers['x-cdn-management-url'] - self.lb_url = self.server_url.replace("servers", "ord.loadbalancers") - self.auth_token = headers['x-auth-token'] - except KeyError: + if resp.status == httplib.NO_CONTENT: + # HTTP NO CONTENT (204): auth successful + headers = dict(resp.getheaders()) + + try: + self.server_url = headers['x-server-management-url'] + self.storage_url = headers['x-storage-url'] + self.cdn_management_url = headers['x-cdn-management-url'] + self.lb_url = self.server_url.replace("servers", "ord.loadbalancers") + self.auth_token = headers['x-auth-token'] + except KeyError, e: + # Returned 204 but has missing information in the header, something is wrong + raise MalformedResponseError('Malformed response', + body='Missing header: %s' % (str(e)), + driver=self.driver) + elif resp.status == httplib.UNAUTHORIZED: + # HTTP UNAUTHORIZED (401): auth failed raise InvalidCredsError() + else: + # Any response code != 401 or 204, something is wrong + raise MalformedResponseError('Malformed response', + body='code: %s body:%s' % (resp.status, ''.join(resp.body.readlines())), + driver=self.driver) for key in ['server_url', 'storage_url', 'cdn_management_url', 'lb_url']: diff --git a/libcloud/compute/base.py b/libcloud/compute/base.py index 65abc932ba..866feac823 100644 --- a/libcloud/compute/base.py +++ b/libcloud/compute/base.py @@ -541,34 +541,34 @@ def deploy_node(self, **kwargs): ssh_username = kwargs.get('ssh_username', 'root') ssh_port = kwargs.get('ssh_port', 22) + ssh_timeout = kwargs.get('ssh_timeout', 20) client = SSHClient(hostname=node.public_ip[0], port=ssh_port, username=ssh_username, password=password, - timeout=kwargs.get('ssh_timeout', None)) - laste = None + timeout=ssh_timeout) + while time.time() < end: - laste = None try: client.connect() - break except (IOError, socket.gaierror, socket.error), e: - laste = e + # Retry if a connection is refused or timeout + # occured + client.close() time.sleep(WAIT_PERIOD) - if laste is not None: - raise e - - tries = 3 - while tries >= 0: - try: - n = kwargs["deploy"].run(node, client) - client.close() - break - except Exception, e: - tries -= 1 - if tries == 0: - raise - client.connect() + continue + + max_tries, tries = 3, 0 + while tries < max_tries: + try: + n = kwargs["deploy"].run(node, client) + client.close() + raise + except Exception, e: + tries += 1 + if tries >= max_tries: + raise DeploymentError(node, + 'Failed after %d tries' % (max_tries)) except DeploymentError: raise diff --git a/libcloud/compute/drivers/dreamhost.py b/libcloud/compute/drivers/dreamhost.py index 59d03f8558..fb857b640a 100644 --- a/libcloud/compute/drivers/dreamhost.py +++ b/libcloud/compute/drivers/dreamhost.py @@ -23,7 +23,6 @@ import copy -from libcloud.pricing import get_pricing from libcloud.common.base import ConnectionKey, Response from libcloud.common.types import InvalidCredsError from libcloud.compute.base import Node, NodeDriver, NodeSize diff --git a/libcloud/compute/drivers/ec2.py b/libcloud/compute/drivers/ec2.py index 7805cd0d65..715dfac1a6 100644 --- a/libcloud/compute/drivers/ec2.py +++ b/libcloud/compute/drivers/ec2.py @@ -1006,10 +1006,17 @@ class NimbusNodeDriver(EC2NodeDriver): _instance_types = NIMBUS_INSTANCE_TYPES def ex_describe_addresses(self, nodes): - """Nimbus doesn't support elastic IPs, so this is a passthrough + """ + Nimbus doesn't support elastic IPs, so this is a passthrough """ nodes_elastic_ip_mappings = {} for node in nodes: # empty list per node nodes_elastic_ip_mappings[node.id] = [] return nodes_elastic_ip_mappings + + def ex_create_tags(self, node, tags): + """ + Nimbus doesn't support creating tags, so this is a passthrough + """ + pass diff --git a/libcloud/compute/drivers/gogrid.py b/libcloud/compute/drivers/gogrid.py index 6f1986bd33..850c0d75d9 100644 --- a/libcloud/compute/drivers/gogrid.py +++ b/libcloud/compute/drivers/gogrid.py @@ -19,11 +19,6 @@ import hashlib import copy -try: - import json -except ImportError: - import simplejson as json - from libcloud.common.types import InvalidCredsError, LibcloudError from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver from libcloud.compute.providers import Provider @@ -66,7 +61,12 @@ 'name': '8GB', 'ram': 8192, 'disk': 480, - 'bandwidth': None} + 'bandwidth': None}, + '16GB': {'id': '16GB', + 'name': '16GB', + 'ram': 16384, + 'disk': 960, + 'bandwidth': None}, } diff --git a/libcloud/compute/drivers/opsource.py b/libcloud/compute/drivers/opsource.py index ad9f48f8c6..d71f1b4efb 100644 --- a/libcloud/compute/drivers/opsource.py +++ b/libcloud/compute/drivers/opsource.py @@ -16,9 +16,7 @@ Opsource Driver """ import base64 -import socket from xml.etree import ElementTree as ET -from xml.parsers.expat import ExpatError from libcloud.utils import fixxpath, findtext, findall from libcloud.common.base import ConnectionUserAndKey, Response @@ -297,9 +295,6 @@ def create_node(self, **kwargs): # XXX: Node sizes can be adjusted after a node is created, but cannot be # set at create time because size is part of the image definition. - size = NodeSize(id=0, name='', ram=0, disk=None, bandwidth=None, - price=0, driver=self.connection.driver) - password = None if kwargs.has_key('auth'): auth = kwargs.get('auth') @@ -330,10 +325,10 @@ def create_node(self, **kwargs): ET.SubElement(server_elm, "administratorPassword").text = password ET.SubElement(server_elm, "isStarted").text = str(ex_isStarted) - data = self.connection.request_with_orgId('server', - method='POST', - data=ET.tostring(server_elm) - ).object + self.connection.request_with_orgId('server', + method='POST', + data=ET.tostring(server_elm) + ).object # XXX: return the last node in the list that has a matching name. this # is likely but not guaranteed to be the node we just created # because opsource allows multiple nodes to have the same name diff --git a/libcloud/compute/drivers/vpsnet.py b/libcloud/compute/drivers/vpsnet.py index edd322ab47..96664284a2 100644 --- a/libcloud/compute/drivers/vpsnet.py +++ b/libcloud/compute/drivers/vpsnet.py @@ -22,8 +22,6 @@ except: import simplejson as json -from libcloud.pricing import get_pricing - from libcloud.common.base import ConnectionUserAndKey, Response from libcloud.common.types import InvalidCredsError from libcloud.compute.providers import Provider diff --git a/libcloud/data/pricing.json b/libcloud/data/pricing.json index 0dd1f2cf67..31554b9c1b 100644 --- a/libcloud/data/pricing.json +++ b/libcloud/data/pricing.json @@ -118,7 +118,8 @@ "1GB": 0.19, "2GB": 0.38, "4GB": 0.76, - "8GB": 1.52 + "8GB": 1.52, + "16GB": 3.04 }, "gandi": { diff --git a/libcloud/drivers/cloudsigma.py b/libcloud/drivers/cloudsigma.py index bf09de855e..f4587fc32d 100644 --- a/libcloud/drivers/cloudsigma.py +++ b/libcloud/drivers/cloudsigma.py @@ -16,7 +16,6 @@ from libcloud.utils import deprecated_warning -from libcloud.utils import str2dicts, str2list, dict2str from libcloud.compute.drivers.cloudsigma import * deprecated_warning(__name__) diff --git a/libcloud/resource/lb/__init__.py b/libcloud/loadbalancer/__init__.py similarity index 100% rename from libcloud/resource/lb/__init__.py rename to libcloud/loadbalancer/__init__.py diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py new file mode 100644 index 0000000000..45b6f1691d --- /dev/null +++ b/libcloud/loadbalancer/base.py @@ -0,0 +1,226 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcloud.common.base import ConnectionKey +from libcloud.common.types import LibcloudError + +__all__ = [ + "Member", + "LoadBalancer", + "Driver", + "Algorithm" + ] + +class Member(object): + + def __init__(self, id, ip, port): + self.id = str(id) if id else None + self.ip = ip + self.port = port + + def __repr__(self): + return ('' % (self.id, + self.ip, self.port)) + +class Algorithm(object): + RANDOM = 0 + ROUND_ROBIN = 1 + LEAST_CONNECTIONS = 2 + +DEFAULT_ALGORITHM = Algorithm.ROUND_ROBIN + +class LoadBalancer(object): + """ + Provide a common interface for handling Load Balancers. + """ + + def __init__(self, id, name, state, ip, port, driver): + self.id = str(id) if id else None + self.name = name + self.state = state + self.ip = ip + self.port = port + self.driver = driver + + def attach_compute_node(self, node): + return self.driver.balancer_attach_compute_node(node) + + def attach_member(self, member): + return self.driver.balancer_attach_member(self, member) + + def detach_member(self, member): + return self.driver.balancer_detach_member(self, member) + + def list_members(self): + return self.driver.balancer_list_members(self) + + def __repr__(self): + return ('' % (self.id, + self.name, self.state)) + + +class Driver(object): + """ + A base LBDriver class to derive from + + This class is always subclassed by a specific driver. + + """ + + connectionCls = ConnectionKey + _ALGORITHM_TO_VALUE_MAP = {} + _VALUE_TO_ALGORITHM_MAP = {} + + def __init__(self, key, secret=None, secure=True): + self.key = key + self.secret = secret + args = [self.key] + + if self.secret is not None: + args.append(self.secret) + + args.append(secure) + + self.connection = self.connectionCls(*args) + self.connection.driver = self + self.connection.connect() + + def list_protocols(self): + """ + Return a list of supported protocols. + """ + + raise NotImplementedError, \ + 'list_protocols not implemented for this driver' + + def list_balancers(self): + """ + List all loadbalancers + + @return: C{list} of L{LoadBalancer} objects + + """ + + raise NotImplementedError, \ + 'list_balancers not implemented for this driver' + + def create_balancer(self, name, port, protocol, algorithm, members): + """ + Create a new load balancer instance + + @keyword name: Name of the new load balancer (required) + @type name: C{str} + @keyword members: C{list} ofL{Member}s to attach to balancer + @type: C{list} of L{Member}s + @keyword protocol: Loadbalancer protocol, defaults to http. + @type: C{str} + @keyword port: Port the load balancer should listen on, defaults to 80 + @type port: C{str} + @keyword algorithm: Load balancing algorithm, defaults to + LBAlgorithm.ROUND_ROBIN + @type algorithm: C{LBAlgorithm} + + """ + + raise NotImplementedError, \ + 'create_balancer not implemented for this driver' + + def destroy_balancer(self, balancer): + """Destroy a load balancer + + @return: C{bool} True if the destroy was successful, otherwise False + + """ + + raise NotImplementedError, \ + 'destroy_balancer not implemented for this driver' + + def get_balancer(self, balancer_id): + """ + Return a C{LoadBalancer} object. + + @keyword balancer_id: id of a load balancer you want to fetch + @type balancer_id: C{str} + + @return: C{LoadBalancer} + """ + + raise NotImplementedError, \ + 'get_balancer not implemented for this driver' + + def balancer_attach_compute_node(self, balancer, node): + """ + Attach a compute node as a member to the load balancer. + + @keyword node: Member to join to the balancer + @type member: C{libcloud.compute.base.Node} + @return {Member} Member after joining the balancer. + """ + + return self.attach_member(Member(None, node.public_ip[0], balancer.port)) + + def balancer_attach_member(self, balancer, member): + """ + Attach a member to balancer + + @keyword member: Member to join to the balancer + @type member: C{Member} + @return {Member} Member after joining the balancer. + """ + + raise NotImplementedError, \ + 'balancer_attach_member not implemented for this driver' + + def balancer_detach_member(self, balancer, member): + """ + Detach member from balancer + + @return: C{bool} True if member detach was successful, otherwise False + + """ + + raise NotImplementedError, \ + 'balancer_detach_member not implemented for this driver' + + def balancer_list_members(self, balancer): + """ + Return list of members attached to balancer + + @return: C{list} of L{Member}s + + """ + + raise NotImplementedError, \ + 'balancer_list_members not implemented for this driver' + + def _value_to_algorithm(self, value): + """ + Return C{LBAlgorithm} based on the value. + """ + try: + return self._VALUE_TO_ALGORITHM_MAP[value] + except KeyError: + raise LibcloudError(value='Invalid value: %s' % (value), + driver=self) + + def _algorithm_to_value(self, algorithm): + """ + Return value based in the algorithm (C{LBAlgorithm}). + """ + try: + return self._ALGORITHM_TO_VALUE_MAP[algorithm] + except KeyError: + raise LibcloudError(value='Invalid algorithm: %s' % (algorithm), + driver=self) diff --git a/libcloud/resource/lb/drivers/__init__.py b/libcloud/loadbalancer/drivers/__init__.py similarity index 100% rename from libcloud/resource/lb/drivers/__init__.py rename to libcloud/loadbalancer/drivers/__init__.py diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py new file mode 100644 index 0000000000..6ecc4c0e4a --- /dev/null +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -0,0 +1,217 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import httplib + +try: + import json +except ImportError: + import simplejson as json + +from libcloud.common.types import LibcloudError +from libcloud.utils import reverse_dict +from libcloud.common.gogrid import GoGridConnection, GoGridResponse, BaseGoGridDriver +from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm +from libcloud.loadbalancer.base import DEFAULT_ALGORITHM +from libcloud.loadbalancer.types import State, LibcloudLBImmutableError + +class GoGridLBResponse(GoGridResponse): + def success(self): + if self.status == httplib.INTERNAL_SERVER_ERROR: + # Hack, but at least this error message is more useful than + # "unexpected server error" + body = json.loads(self.body) + if body['method'] == '/grid/loadbalancer/add' and \ + len(body['list']) >= 1 and \ + body['list'][0]['message'].find('unexpected server error') != -1: + raise LibcloudError(value='You mostly likely tried to add a ' + + 'member with an IP address not assigned ' + + 'to your account', driver=self) + return super(GoGridLBResponse, self).success() + +class GoGridLBConnection(GoGridConnection): + """ + Connection class for the GoGrid load-balancer driver. + """ + responseCls = GoGridLBResponse + +class GoGridLBDriver(BaseGoGridDriver, Driver): + connectionCls = GoGridLBConnection + api_name = 'gogrid_lb' + name = 'GoGrid LB' + + LB_STATE_MAP = { 'On': State.RUNNING, + 'Unknown': State.UNKNOWN } + _VALUE_TO_ALGORITHM_MAP = { + 'round robin': Algorithm.ROUND_ROBIN, + 'least connect': Algorithm.LEAST_CONNECTIONS + } + _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) + + def list_protocols(self): + # GoGrid only supports http + return [ 'http' ] + + def list_balancers(self): + return self._to_balancers( + self.connection.request('/api/grid/loadbalancer/list').object) + + def ex_create_balancer_nowait(self, name, members, protocol='http', port=80, + algorithm=DEFAULT_ALGORITHM): + algorithm = self._algorithm_to_value(algorithm) + + params = {'name': name, + 'loadbalancer.type': algorithm, + 'virtualip.ip': self._get_first_ip(), + 'virtualip.port': port} + params.update(self._members_to_params(members)) + + resp = self.connection.request('/api/grid/loadbalancer/add', + method='GET', + params=params) + return self._to_balancers(resp.object)[0] + + def create_balancer(self, name, members, protocol='http', port=80, + algorithm=DEFAULT_ALGORITHM): + balancer = self.ex_create_balancer_nowait(name, members, protocol, + port, algorithm) + + timeout = 60 * 20 + waittime = 0 + interval = 2 * 15 + + if balancer.id is not None: + return balancer + else: + while waittime < timeout: + balancers = self.list_balancers() + + for i in balancers: + if i.name == balancer.name and i.id is not None: + return i + + waittime += interval + time.sleep(interval) + + raise Exception('Failed to get id') + + def destroy_balancer(self, balancer): + try: + resp = self.connection.request('/api/grid/loadbalancer/delete', + method='POST', params={'id': balancer.id}) + except Exception as err: + if "Update request for LoadBalancer" in str(err): + raise LibcloudLBImmutableError("Cannot delete immutable object", + GoGridLBDriver) + else: + raise + + return resp.status == 200 + + def get_balancer(self, **kwargs): + params = {} + + try: + params['name'] = kwargs['ex_balancer_name'] + except KeyError: + balancer_id = kwargs['balancer_id'] + params['id'] = balancer_id + + resp = self.connection.request('/api/grid/loadbalancer/get', + params=params) + + return self._to_balancers(resp.object)[0] + + def balancer_attach_member(self, balancer, member): + members = self.balancer_list_members(balancer) + members.append(member) + + params = {"id": balancer.id} + + params.update(self._members_to_params(members)) + + resp = self._update_balancer(params) + + return [ m for m in + self._to_members(resp.object["list"][0]["realiplist"]) + if m.ip == member.ip ][0] + + def balancer_detach_member(self, balancer, member): + members = self.balancer_list_members(balancer) + + remaining_members = [n for n in members if n.id != member.id] + + params = {"id": balancer.id} + params.update(self._members_to_params(remaining_members)) + + resp = self._update_balancer(params) + + return resp.status == 200 + + def balancer_list_members(self, balancer): + resp = self.connection.request('/api/grid/loadbalancer/get', + params={'id': balancer.id}) + return self._to_members(resp.object["list"][0]["realiplist"]) + + def _update_balancer(self, params): + try: + return self.connection.request('/api/grid/loadbalancer/edit', + method='POST', + params=params) + except Exception as err: + if "Update already pending" in str(err): + raise LibcloudLBImmutableError("Balancer is immutable", GoGridLBDriver) + + raise LibcloudError(value='Exception: %s' % str(err), driver=self) + + def _members_to_params(self, members): + """ + Helper method to convert list of L{Member} objects + to GET params. + + """ + + params = {} + + i = 0 + for member in members: + params["realiplist.%s.ip" % i] = member.ip + params["realiplist.%s.port" % i] = member.port + i += 1 + + return params + + def _to_balancers(self, object): + return [ self._to_balancer(el) for el in object["list"] ] + + def _to_balancer(self, el): + lb = LoadBalancer(id=el.get("id"), + name=el["name"], + state=self.LB_STATE_MAP.get( + el["state"]["name"], State.UNKNOWN), + ip=el["virtualip"]["ip"]["ip"], + port=el["virtualip"]["port"], + driver=self.connection.driver) + return lb + + def _to_members(self, object): + return [ self._to_member(el) for el in object ] + + def _to_member(self, el): + member = Member(id=el["ip"]["id"], + ip=el["ip"]["ip"], + port=el["port"]) + return member diff --git a/libcloud/resource/lb/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py similarity index 63% rename from libcloud/resource/lb/drivers/rackspace.py rename to libcloud/loadbalancer/drivers/rackspace.py index 6e645357ab..e82e91ed79 100644 --- a/libcloud/resource/lb/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -18,11 +18,13 @@ try: import json except ImportError: - import simplejson + import simplejson as json +from libcloud.utils import reverse_dict from libcloud.common.base import Response -from libcloud.resource.lb.base import LB, LBNode, LBDriver -from libcloud.resource.lb.types import Provider, LBState +from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm +from libcloud.loadbalancer.base import DEFAULT_ALGORITHM +from libcloud.loadbalancer.types import Provider, State from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) @@ -63,32 +65,41 @@ def request(self, action, params=None, data='', headers=None, method='GET'): params=params, data=data, method=method, headers=headers) -class RackspaceLBDriver(LBDriver): +class RackspaceLBDriver(Driver): connectionCls = RackspaceConnection - type = Provider.RACKSPACE api_name = 'rackspace_lb' name = 'Rackspace LB' - LB_STATE_MAP = { 'ACTIVE': LBState.RUNNING, - 'BUILD': LBState.PENDING } + LB_STATE_MAP = { 'ACTIVE': State.RUNNING, + 'BUILD': State.PENDING } + _VALUE_TO_ALGORITHM_MAP = { + 'RANDOM': Algorithm.RANDOM, + 'ROUND_ROBIN': Algorithm.ROUND_ROBIN, + 'LEAST_CONNECTIONS': Algorithm.LEAST_CONNECTIONS + } + _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) + + def list_protocols(self): + return self._to_protocols( + self.connection.request('/loadbalancers/protocols').object) def list_balancers(self): return self._to_balancers( self.connection.request('/loadbalancers').object) - def create_balancer(self, **kwargs): - name = kwargs['name'] - port = kwargs['port'] - nodes = kwargs['nodes'] + def create_balancer(self, name, members, protocol='http', + port=80, algorithm=DEFAULT_ALGORITHM): + algorithm = self._algorithm_to_value(algorithm) balancer_object = {"loadBalancer": {"name": name, "port": port, - "protocol": "HTTP", + "algorithm": algorithm, + "protocol": protocol.upper(), "virtualIps": [{"type": "PUBLIC"}], - "nodes": [{"address": node.ip, - "port": node.port, - "condition": "ENABLED"} for node in nodes], + "nodes": [{"address": member.ip, + "port": member.port, + "condition": "ENABLED"} for member in members], } } @@ -103,22 +114,17 @@ def destroy_balancer(self, balancer): return resp.status == 202 - def balancer_detail(self, **kwargs): - try: - balancer_id = kwargs['balancer_id'] - except KeyError: - balancer_id = kwargs['balancer'].id - + def get_balancer(self, balancer_id): uri = '/loadbalancers/%s' % (balancer_id) resp = self.connection.request(uri) return self._to_balancer(resp.object["loadBalancer"]) - def balancer_attach_node(self, balancer, **kwargs): - ip = kwargs['ip'] - port = kwargs['port'] + def balancer_attach_member(self, balancer, member): + ip = member.ip + port = member.port - node_object = {"nodes": + member_object = {"nodes": [{"port": port, "address": ip, "condition": "ENABLED"}] @@ -126,38 +132,47 @@ def balancer_attach_node(self, balancer, **kwargs): uri = '/loadbalancers/%s/nodes' % (balancer.id) resp = self.connection.request(uri, method='POST', - data=json.dumps(node_object)) - return self._to_nodes(resp.object)[0] - - def balancer_detach_node(self, balancer, node): - uri = '/loadbalancers/%s/nodes/%s' % (balancer.id, node.id) + data=json.dumps(member_object)) + return self._to_members(resp.object)[0] + + def balancer_detach_member(self, balancer, member): + # Loadbalancer always needs to have at least 1 member. + # Last member cannot be detached. You can only disable it or destroy the + # balancer. + uri = '/loadbalancers/%s/nodes/%s' % (balancer.id, member.id) resp = self.connection.request(uri, method='DELETE') return resp.status == 202 - def balancer_list_nodes(self, balancer): + def balancer_list_members(self, balancer): uri = '/loadbalancers/%s/nodes' % (balancer.id) - return self._to_nodes( + return self._to_members( self.connection.request(uri).object) + def _to_protocols(self, object): + protocols = [] + for item in object["protocols"]: + protocols.append(item['name'].lower()) + return protocols + def _to_balancers(self, object): return [ self._to_balancer(el) for el in object["loadBalancers"] ] def _to_balancer(self, el): - lb = LB(id=el["id"], + lb = LoadBalancer(id=el["id"], name=el["name"], state=self.LB_STATE_MAP.get( - el["status"], LBState.UNKNOWN), + el["status"], State.UNKNOWN), ip=el["virtualIps"][0]["address"], port=el["port"], driver=self.connection.driver) return lb - def _to_nodes(self, object): - return [ self._to_node(el) for el in object["nodes"] ] + def _to_members(self, object): + return [ self._to_member(el) for el in object["nodes"] ] - def _to_node(self, el): - lbnode = LBNode(id=el["id"], + def _to_member(self, el): + lbmember = Member(id=el["id"], ip=el["address"], port=el["port"]) - return lbnode + return lbmember diff --git a/libcloud/resource/lb/providers.py b/libcloud/loadbalancer/providers.py similarity index 81% rename from libcloud/resource/lb/providers.py rename to libcloud/loadbalancer/providers.py index 48162c48ce..fb12e82813 100644 --- a/libcloud/resource/lb/providers.py +++ b/libcloud/loadbalancer/providers.py @@ -14,7 +14,7 @@ # limitations under the License. from libcloud.utils import get_driver as get_provider_driver -from libcloud.resource.lb.types import Provider +from libcloud.loadbalancer.types import Provider __all__ = [ "Provider", @@ -23,10 +23,10 @@ ] DRIVERS = { - Provider.RACKSPACE: - ('libcloud.resource.lb.drivers.rackspace', 'RackspaceLBDriver'), + Provider.RACKSPACE_US: + ('libcloud.loadbalancer.drivers.rackspace', 'RackspaceLBDriver'), Provider.GOGRID: - ('libcloud.resource.lb.drivers.gogrid', 'GoGridLBDriver'), + ('libcloud.loadbalancer.drivers.gogrid', 'GoGridLBDriver'), } def get_driver(provider): diff --git a/libcloud/resource/lb/types.py b/libcloud/loadbalancer/types.py similarity index 95% rename from libcloud/resource/lb/types.py rename to libcloud/loadbalancer/types.py index 143ca6f802..79c214436b 100644 --- a/libcloud/resource/lb/types.py +++ b/libcloud/loadbalancer/types.py @@ -15,7 +15,7 @@ __all__ = [ "Provider", - "LBState", + "State", "LibcloudLBError", "LibcloudLBImmutableError", ] @@ -27,10 +27,10 @@ class LibcloudLBError(LibcloudError): pass class LibcloudLBImmutableError(LibcloudLBError): pass class Provider(object): - RACKSPACE = 0 + RACKSPACE_US = 0 GOGRID = 1 -class LBState(object): +class State(object): """ Standart states for a loadbalancer diff --git a/libcloud/resource/lb/base.py b/libcloud/resource/lb/base.py deleted file mode 100644 index a293cd217d..0000000000 --- a/libcloud/resource/lb/base.py +++ /dev/null @@ -1,175 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from libcloud.common.base import ConnectionKey - -__all__ = [ - "LBNode", - "LB", - "LBDriver", - ] - -class LBNode(object): - - def __init__(self, id, ip, port): - self.id = str(id) if id else None - self.ip = ip - self.port = port - - def __repr__(self): - return ('' % (self.id, - self.ip, self.port)) - - -class LB(object): - """ - Provide a common interface for handling Load Balancers. - """ - - def __init__(self, id, name, state, ip, port, driver): - self.id = str(id) if id else None - self.name = name - self.state = state - self.ip = ip - self.port = port - self.driver = driver - - def attach_node(self, **kwargs): - return self.driver.balancer_attach_node(self, **kwargs) - - def detach_node(self, node): - return self.driver.balancer_detach_node(self, node) - - def list_nodes(self): - return self.driver.balancer_list_nodes(self) - - def __repr__(self): - return ('' % (self.id, - self.name, self.state)) - - -class LBDriver(object): - """ - A base LBDriver class to derive from - - This class is always subclassed by a specific driver. - - """ - - connectionCls = ConnectionKey - - def __init__(self, key, secret=None, secure=True): - self.key = key - self.secret = secret - args = [self.key] - - if self.secret is not None: - args.append(self.secret) - - args.append(secure) - - self.connection = self.connectionCls(*args) - self.connection.driver = self - self.connection.connect() - - def list_balancers(self): - """ - List all loadbalancers - - @return: C{list} of L{LB} objects - - """ - - raise NotImplementedError, \ - 'list_balancers not implemented for this driver' - - def create_balancer(self, **kwargs): - """ - Create a new load balancer instance - - @keyword name: Name of the new load balancer (required) - @type name: C{str} - @keyword port: Port the load balancer should listen on (required) - @type port: C{str} - @keyword nodes: C{list} of L{LBNode}s to attach to balancer - @type: C{list} of L{LBNode}s - - """ - - raise NotImplementedError, \ - 'create_balancer not implemented for this driver' - - def destroy_balancer(self, balancer): - """Destroy a load balancer - - @return: C{bool} True if the destroy was successful, otherwise False - - """ - - raise NotImplementedError, \ - 'destroy_balancer not implemented for this driver' - - def balancer_detail(self, **kwargs): - """ - Returns a detailed info about load balancer given by - existing L{LB} object or its id - - @keyword balancer: L{LB} object you already fetched using list method for example - @type balancer: L{LB} - @keyword balancer_id: id of a load balancer you want to fetch - @type balancer_id: C{str} - - @return: L{LB} - - """ - - raise NotImplementedError, \ - 'balancer_detail not implemented for this driver' - - def balancer_attach_node(self, balancer, **kwargs): - """ - Attach a node to balancer - - @keyword ip: IP address of a node - @type ip: C{str} - @keyword port: port that services we're balancing listens on on the node - @keyword port: C{str} - - """ - - raise NotImplementedError, \ - 'balancer_attach_node not implemented for this driver' - - def balancer_detach_node(self, balancer, node): - """ - Detach node from balancer - - @return: C{bool} True if node detach was successful, otherwise False - - """ - - raise NotImplementedError, \ - 'balancer_detach_node not implemented for this driver' - - def balancer_list_nodes(self, balancer): - """ - Return list of nodes attached to balancer - - @return: C{list} of L{LBNode}s - - """ - - raise NotImplementedError, \ - 'balancer_list_nodes not implemented for this driver' diff --git a/libcloud/resource/lb/drivers/gogrid.py b/libcloud/resource/lb/drivers/gogrid.py deleted file mode 100644 index 1e41b5b388..0000000000 --- a/libcloud/resource/lb/drivers/gogrid.py +++ /dev/null @@ -1,193 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import time - -try: - import json -except ImportError: - import simplejson - -from libcloud.common.types import LibcloudError -from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver -from libcloud.resource.lb.base import LB, LBNode, LBDriver -from libcloud.resource.lb.types import Provider, LBState, LibcloudLBImmutableError - - -class GoGridLBDriver(BaseGoGridDriver, LBDriver): - connectionCls = GoGridConnection - type = Provider.RACKSPACE - api_name = 'gogrid_lb' - name = 'GoGrid LB' - - LB_STATE_MAP = { 'On': LBState.RUNNING, - 'Unknown': LBState.UNKNOWN } - - def list_balancers(self): - return self._to_balancers( - self.connection.request('/api/grid/loadbalancer/list').object) - - def ex_create_balancer_nowait(self, **kwargs): - name = kwargs['name'] - port = kwargs['port'] - nodes = kwargs['nodes'] - - params = {'name': name, - 'virtualip.ip': self._get_first_ip(), - 'virtualip.port': port} - params.update(self._nodes_to_params(nodes)) - - resp = self.connection.request('/api/grid/loadbalancer/add', - method='GET', - params=params) - return self._to_balancers(resp.object)[0] - - def create_balancer(self, **kwargs): - balancer = self.ex_create_balancer_nowait(**kwargs) - - timeout = 60 * 20 - waittime = 0 - interval = 2 * 15 - - if balancer.id is not None: - return balancer - else: - while waittime < timeout: - balancers = self.list_balancers() - - for i in balancers: - if i.name == balancer.name and i.id is not None: - return i - - waittime += interval - time.sleep(interval) - - raise Exception('Failed to get id') - - def destroy_balancer(self, balancer): - try: - resp = self.connection.request('/api/grid/loadbalancer/delete', - method='POST', params={'id': balancer.id}) - except Exception as err: - if "Update request for LoadBalancer" in str(err): - raise LibcloudLBImmutableError("Cannot delete immutable object", - GoGridLBDriver) - else: - raise - - return resp.status == 200 - - def balancer_detail(self, **kwargs): - params = {} - - try: - params['name'] = kwargs['balancer_name'] - except KeyError: - try: - balancer_id = kwargs['balancer_id'] - except KeyError: - balancer_id = kwargs['balancer'].id - - params['id'] = balancer_id - - resp = self.connection.request('/api/grid/loadbalancer/get', - params=params) - - return self._to_balancers(resp.object)[0] - - def balancer_attach_node(self, balancer, **kwargs): - ip = kwargs['ip'] - port = kwargs['port'] - - nodes = self.balancer_list_nodes(balancer) - nodes.append(LBNode(None, ip, port)) - - params = {"id": balancer.id} - - params.update(self._nodes_to_params(nodes)) - - resp = self._update_node(params) - - return [ node for node in - self._to_nodes(resp.object["list"][0]["realiplist"]) - if node.ip == ip ][0] - - def balancer_detach_node(self, balancer, node): - nodes = self.balancer_list_nodes(balancer) - - remaining_nodes = [n for n in nodes if n.id != node.id] - - params = {"id": balancer.id} - params.update(self._nodes_to_params(remaining_nodes)) - - resp = self._update_node(params) - - return resp.status == 200 - - def balancer_list_nodes(self, balancer): - resp = self.connection.request('/api/grid/loadbalancer/get', - params={'id': balancer.id}) - return self._to_nodes(resp.object["list"][0]["realiplist"]) - - def _update_node(self, params): - try: - return self.connection.request('/api/grid/loadbalancer/edit', - method='POST', - params=params) - except Exception as err: - if "Update already pending" in str(err): - raise LibcloudLBImmutableError("Balancer is immutable", GoGridLBDriver) - - raise LibcloudError(value='Exception: %s' % str(err), driver=self) - - def _nodes_to_params(self, nodes): - """ - Helper method to convert list of L{LBNode} objects - to GET params. - - """ - - params = {} - - i = 0 - for node in nodes: - params["realiplist.%s.ip" % i] = node.ip - params["realiplist.%s.port" % i] = node.port - i += 1 - - return params - - def _to_balancers(self, object): - return [ self._to_balancer(el) for el in object["list"] ] - - def _to_balancer(self, el): - lb = LB(id=el.get("id"), - name=el["name"], - state=self.LB_STATE_MAP.get( - el["state"]["name"], LBState.UNKNOWN), - ip=el["virtualip"]["ip"]["ip"], - port=el["virtualip"]["port"], - driver=self.connection.driver) - return lb - - def _to_nodes(self, object): - return [ self._to_node(el) for el in object ] - - def _to_node(self, el): - lbnode = LBNode(id=el["ip"]["id"], - ip=el["ip"]["ip"], - port=el["port"]) - return lbnode diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index ebde0e00ee..b7dd09e99c 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -122,9 +122,9 @@ def get_object(self, object_name): return self.driver.get_object(container_name=self.name, object_name=object_name) - def upload_object(self, file_path, object_name, extra=None, file_hash=None): + def upload_object(self, file_path, object_name, extra=None, verify_hash=True): return self.driver.upload_object( - file_path, self, object_name, extra, file_hash) + file_path, self, object_name, extra, verify_hash) def upload_object_via_stream(self, iterator, object_name, extra=None): return self.driver.upload_object_via_stream( @@ -178,16 +178,6 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None): self.connection.driver = self self.connection.connect() - def get_meta_data(self): - """ - Return account meta data - total number of containers, objects and - number of bytes currently used. - - @return A C{dict} with account meta data. - """ - raise NotImplementedError( - 'get_account_meta_data not implemented for this driver') - def list_containters(self): raise NotImplementedError( 'list_containers not implemented for this driver') @@ -263,7 +253,7 @@ def enable_object_cdn(self, obj): raise NotImplementedError( 'enable_object_cdn not implemented for this driver') - def download_object(self, obj, destination_path, delete_on_failure=True): + def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): """ Download an object to the specified destination path. @@ -275,7 +265,7 @@ def download_object(self, obj, destination_path, delete_on_failure=True): incoming file will be saved. @type overwrite_existing: C{bool} - @type overwrite_existing: True to overwrite an existing file. + @type overwrite_existing: True to overwrite an existing file, defaults to False. @type delete_on_failure: C{bool} @param delete_on_failure: True to delete a partially downloaded file if @@ -301,7 +291,7 @@ def download_object_as_stream(self, obj, chunk_size=None): 'download_object_as_stream not implemented for this driver') def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None): + verify_hash=True): """ Upload an object. @@ -317,10 +307,8 @@ def upload_object(self, file_path, container, object_name, extra=None, @type extra: C{dict} @param extra: (optional) Extra attributes (driver specific). - @type file_hash: C{str} - @param file_hash: (optional) File hash. If provided object hash is - on upload and if it doesn't match the one provided an - exception is thrown. + @type verify_hash: C{boolean} + @param verify_hash: True to do a file integrity check. """ raise NotImplementedError( 'upload_object not implemented for this driver') @@ -438,7 +426,7 @@ def _save_object(self, response, obj, destination_path, exists. @type chunk_size: C{int} - @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) + @param chunk_size: Optional chunk size (defaults to L{libcloud.storage.base.CHUNK_SIZE}, 8kb) @return C{bool} True on success, False otherwise. """ diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index b4c8ff0197..0f882f1e35 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -21,10 +21,9 @@ except: import simplejson as json -from libcloud.utils import fixxpath, findtext, in_development_warning from libcloud.utils import read_in_chunks from libcloud.common.types import MalformedResponseError, LibcloudError -from libcloud.common.base import Response +from libcloud.common.base import Response, RawResponse from libcloud.storage.providers import Provider from libcloud.storage.base import Object, Container, StorageDriver @@ -79,6 +78,8 @@ def parse_body(self): return data +class CloudFilesRawResponse(CloudFilesResponse, RawResponse): + pass class CloudFilesConnection(RackspaceBaseConnection): """ @@ -86,6 +87,7 @@ class CloudFilesConnection(RackspaceBaseConnection): """ responseCls = CloudFilesResponse + rawResponseCls = CloudFilesRawResponse auth_host = None _url_key = "storage_url" @@ -148,23 +150,6 @@ class CloudFilesStorageDriver(StorageDriver): connectionCls = CloudFilesConnection hash_type = 'md5' - def get_meta_data(self): - response = self.connection.request('', method='HEAD') - - if response.status == httplib.NO_CONTENT: - container_count = response.headers.get( - 'x-account-container-count', 'unknown') - object_count = response.headers.get( - 'x-account-object-count', 'unknown') - bytes_used = response.headers.get( - 'x-account-bytes-used', 'unknown') - - return { 'container_count': int(container_count), - 'object_count': int(object_count), - 'bytes_used': int(bytes_used) } - - raise LibcloudError('Unexpected status code: %s' % (response.status)) - def list_containers(self): response = self.connection.request('') @@ -204,7 +189,6 @@ def get_object(self, container_name, object_name): response = self.connection.request('/%s/%s' % (container_name, object_name), method='HEAD') - if response.status in [ httplib.OK, httplib.NO_CONTENT ]: obj = self._headers_to_object( object_name, container, response.headers) @@ -310,7 +294,7 @@ def download_object_as_stream(self, obj, chunk_size=None): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None): + verify_hash=True): """ Upload an object. @@ -323,7 +307,7 @@ def upload_object(self, file_path, container, object_name, extra=None, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - file_hash=file_hash) + verify_hash=verify_hash) def upload_object_via_stream(self, iterator, container, object_name, extra=None): @@ -353,9 +337,26 @@ def delete_object(self, obj): raise LibcloudError('Unexpected status code: %s' % (response.status)) + def ex_get_meta_data(self): + response = self.connection.request('', method='HEAD') + + if response.status == httplib.NO_CONTENT: + container_count = response.headers.get( + 'x-account-container-count', 'unknown') + object_count = response.headers.get( + 'x-account-object-count', 'unknown') + bytes_used = response.headers.get( + 'x-account-bytes-used', 'unknown') + + return { 'container_count': int(container_count), + 'object_count': int(object_count), + 'bytes_used': int(bytes_used) } + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, file_hash=None): + iterator=None, verify_hash=True): extra = extra or {} container_name_cleaned = self._clean_container_name(container.name) object_name_cleaned = self._clean_object_name(object_name) @@ -363,9 +364,6 @@ def _put_object(self, container, object_name, upload_func, meta_data = extra.get('meta_data', None) headers = {} - if not iterator and file_hash: - headers['ETag'] = file_hash - if meta_data: for key, value in meta_data.iteritems(): key = 'X-Object-Meta-%s' % (key) @@ -383,17 +381,22 @@ def _put_object(self, container, object_name, upload_func, response = result_dict['response'].response bytes_transferred = result_dict['bytes_transferred'] + server_hash = result_dict['response'].headers.get('etag', None) if response.status == httplib.EXPECTATION_FAILED: raise LibcloudError(value='Missing content-type header', driver=self) - elif response.status == httplib.UNPROCESSABLE_ENTITY: + elif verify_hash and not server_hash: + raise LibcloudError(value='Server didn\'t return etag', + driver=self) + elif (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( - value='MD5 hash checksum does not match', + value=('MD5 hash checksum does not match (expected=%s, ' + + 'actual=%s)') % (result_dict['data_hash'], server_hash), object_name=object_name, driver=self) elif response.status == httplib.CREATED: obj = Object( - name=object_name, size=bytes_transferred, hash=file_hash, + name=object_name, size=bytes_transferred, hash=server_hash, extra=None, meta_data=meta_data, container=container, driver=self) @@ -476,10 +479,9 @@ def _headers_to_object(self, name, container, headers): key = key.replace('x-object-meta-', '') meta_data[key] = value - extra = { 'content_type': content_type, 'last_modified': last_modified, - 'etag': etag } + extra = { 'content_type': content_type, 'last_modified': last_modified } - obj = Object(name=name, size=size, hash=None, extra=extra, + obj = Object(name=name, size=size, hash=etag, extra=extra, meta_data=meta_data, container=container, driver=self) return obj diff --git a/libcloud/storage/drivers/dummy.py b/libcloud/storage/drivers/dummy.py index 058e42f8b3..274225d0b9 100644 --- a/libcloud/storage/drivers/dummy.py +++ b/libcloud/storage/drivers/dummy.py @@ -15,6 +15,7 @@ import os.path import random +import hashlib from libcloud.common.types import LibcloudError @@ -48,14 +49,19 @@ def __len__(self): class DummyIterator(object): def __init__(self, data=None): + self.hash = hashlib.md5() self._data = data or [] self._current_item = 0 + def get_md5_hash(self): + return self.hash.hexdigest() + def next(self): if self._current_item == len(self._data): raise StopIteration value = self._data[self._current_item] + self.hash.update(value) self._current_item += 1 return value diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 9ee075a647..12cd9a817c 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -26,7 +26,7 @@ from libcloud.utils import fixxpath, findtext, in_development_warning from libcloud.utils import read_in_chunks from libcloud.common.types import InvalidCredsError, LibcloudError -from libcloud.common.base import ConnectionUserAndKey +from libcloud.common.base import ConnectionUserAndKey, RawResponse from libcloud.common.aws import AWSBaseResponse from libcloud.storage.base import Object, Container, StorageDriver @@ -70,6 +70,9 @@ def parse_error(self): raise LibcloudError('Unknown error. Status code: %d' % (self.status), driver=S3StorageDriver) +class S3RawResponse(S3Response, RawResponse): + pass + class S3Connection(ConnectionUserAndKey): """ Repersents a single connection to the EC2 Endpoint @@ -77,6 +80,7 @@ class S3Connection(ConnectionUserAndKey): host = 's3.amazonaws.com' responseCls = S3Response + rawResponseCls = S3RawResponse def add_default_params(self, params): expires = str(int(time.time()) + EXPIRATION_SECONDS) @@ -114,7 +118,7 @@ def _get_aws_auth_param(self, method, headers, params, expires, if key.lower() in special_header_keys: special_header_values[key.lower()] = value.lower().strip() elif key.lower().startswith('x-amz-'): - amz_header_values[key.lower()] = value.lower().strip() + amz_header_values[key.lower()] = value.strip() if not special_header_values.has_key('content-md5'): special_header_values['content-md5'] = '' @@ -290,7 +294,7 @@ def download_object_as_stream(self, obj, chunk_size=None): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None, ex_storage_class=None): + verify_hash=True, ex_storage_class=None): upload_func = self._upload_file upload_func_kwargs = { 'file_path': file_path } @@ -298,7 +302,7 @@ def upload_object(self, file_path, container, object_name, extra=None, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - file_hash=file_hash, + verify_hash=verify_hash, storage_class=ex_storage_class) def upload_object_via_stream(self, iterator, container, object_name, @@ -328,7 +332,7 @@ def _clean_object_name(self, name): def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, file_hash=None, storage_class=None): + iterator=None, verify_hash=True, storage_class=None): headers = {} extra = extra or {} storage_class = storage_class or 'standard' @@ -342,9 +346,6 @@ def _put_object(self, container, object_name, upload_func, content_type = extra.get('content_type', None) meta_data = extra.get('meta_data', None) - if not iterator and file_hash: - headers['Content-MD5'] = base64.b64encode(file_hash.decode('hex')) - if meta_data: for key, value in meta_data.iteritems(): key = 'x-amz-meta-%s' % (key) @@ -368,19 +369,22 @@ def _put_object(self, container, object_name, upload_func, bytes_transferred = result_dict['bytes_transferred'] headers = response.headers response = response.response + server_hash = headers['etag'].replace('"', '') - if (file_hash and response.status == httplib.BAD_REQUEST) or \ - (file_hash and file_hash != headers['etag'].replace('"', '')): + if (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( value='MD5 hash checksum does not match', object_name=object_name, driver=self) elif response.status == httplib.OK: obj = Object( - name=object_name, size=bytes_transferred, hash=file_hash, + name=object_name, size=bytes_transferred, hash=server_hash, extra=None, meta_data=meta_data, container=container, driver=self) return obj + else: + raise LibcloudError('Unexpected status code, status_code=%s' % (response.status), + driver=self) def _to_containers(self, obj, xpath): return [ self._to_container(element) for element in \ diff --git a/libcloud/storage/drivers/swift.py b/libcloud/storage/drivers/swift.py new file mode 100644 index 0000000000..16d7fe5b10 --- /dev/null +++ b/libcloud/storage/drivers/swift.py @@ -0,0 +1,450 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import httplib +import urllib + +try: + import json +except: + import simplejson as json + +from libcloud.utils import read_in_chunks +from libcloud.common.types import MalformedResponseError, LibcloudError +from libcloud.common.base import Response, RawResponse + +from libcloud.storage.providers import Provider +from libcloud.storage.base import Object, Container, StorageDriver +from libcloud.storage.types import ContainerAlreadyExistsError +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ContainerIsNotEmptyError +from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError +from libcloud.storage.types import InvalidContainerNameError + +from libcloud.common.openstack import OpenstackBaseConnection + +API_VERSION = 'v1.0' + +class SwiftResponse(Response): + + valid_response_codes = [ httplib.NOT_FOUND, httplib.CONFLICT ] + + def success(self): + i = int(self.status) + return i >= 200 and i <= 299 or i in self.valid_response_codes + + def parse_body(self): + if not self.body: + return None + + if 'content-type' in self.headers: + key = 'content-type' + elif 'Content-Type' in self.headers: + key = 'Content-Type' + else: + raise LibcloudError('Missing content-type header') + + content_type = self.headers[key] + if content_type.find(';') != -1: + content_type = content_type.split(';')[0] + + if content_type == 'application/json': + try: + data = json.loads(self.body) + except: + raise MalformedResponseError('Failed to parse JSON', + body=self.body, + driver=SwiftStorageDriver) + elif content_type == 'text/plain': + data = self.body + else: + data = self.body + + return data + +class SwiftRawResponse(SwiftResponse, RawResponse): + pass + +class SwiftConnection(OpenstackBaseConnection): + """ + Base connection class for the Cloudfiles driver. + """ + + responseCls = SwiftResponse + rawResponseCls = SwiftRawResponse + auth_host = None + _url_key = "storage_url" + + def __init__(self, user_id, key, secure=True): + super(SwiftConnection, self).__init__(user_id, key, secure=secure) + self.api_version = API_VERSION + self.accept_format = 'application/json' + + def request(self, action, params=None, data='', headers=None, method='GET', raw=False): + if not headers: + headers = {} + if not params: + params = {} + + # Due to first-run authentication request, we may not have a path + if self.request_path: + action = self.request_path + action + params['format'] = 'json' + if method in [ 'POST', 'PUT' ]: + headers.update({'Content-Type': 'application/json; charset=UTF-8'}) + + return super(SwiftConnection, self).request( + action=action, + params=params, data=data, + method=method, headers=headers, + raw=raw + ) + +class SwiftOSConnection(SwiftConnection): + """ + Connection class for the Cloudfiles US endpoint. + """ + def __init__(self, user_id, key, host, port, secure=True): + super(SwiftOSConnection, self).__init__(user_id,key, secure=secure) + self.auth_host = host + self.port = (port,port) + + +class SwiftStorageDriver(StorageDriver): + """ + Base Swift driver. + + You should never create an instance of this class directly but use US/US + class. + """ + name = 'Swift' + connectionCls = SwiftConnection + hash_type = 'md5' + + def list_containers(self): + response = self.connection.request('') + + if response.status == httplib.NO_CONTENT: + return [] + elif response.status == httplib.OK: + return self._to_container_list(json.loads(response.body)) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def list_container_objects(self, container): + response = self.connection.request('/%s' % (container.name)) + + if response.status == httplib.NO_CONTENT: + # Empty or inexistent container + return [] + elif response.status == httplib.OK: + return self._to_object_list(json.loads(response.body), container) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def get_container(self, container_name): + response = self.connection.request('/%s' % (container_name), + method='HEAD') + + if response.status == httplib.NO_CONTENT: + container = self._headers_to_container( + container_name, response.headers) + return container + elif response.status == httplib.NOT_FOUND: + raise ContainerDoesNotExistError(None, self, container_name) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def get_object(self, container_name, object_name): + container = self.get_container(container_name) + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='HEAD') + if response.status in [ httplib.OK, httplib.NO_CONTENT ]: + obj = self._headers_to_object( + object_name, container, response.headers) + return obj + elif response.status == httplib.NOT_FOUND: + raise ObjectDoesNotExistError(None, self, object_name) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def create_container(self, container_name): + container_name = self._clean_container_name(container_name) + response = self.connection.request( + '/%s' % (container_name), method='PUT') + + if response.status == httplib.CREATED: + # Accepted mean that container is not yet created but it will be + # eventually + extra = { 'object_count': 0 } + container = Container(name=container_name, extra=extra, driver=self) + + return container + elif response.status == httplib.ACCEPTED: + error = ContainerAlreadyExistsError(None, self, container_name) + raise error + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def delete_container(self, container): + name = self._clean_container_name(container.name) + + # Only empty container can be deleted + response = self.connection.request('/%s' % (name), method='DELETE') + + if response.status == httplib.NO_CONTENT: + return True + elif response.status == httplib.NOT_FOUND: + raise ContainerDoesNotExistError(value='', + container_name=name, driver=self) + elif response.status == httplib.CONFLICT: + # @TODO: Add "delete_all_objects" parameter? + raise ContainerIsNotEmptyError(value='', + container_name=name, driver=self) + + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + container_name = obj.container.name + object_name = obj.name + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='GET', raw=True) + + return self._get_object(obj=obj, callback=self._save_object, + response=response, + callback_kwargs={'obj': obj, + 'response': response.response, + 'destination_path': destination_path, + 'overwrite_existing': overwrite_existing, + 'delete_on_failure': delete_on_failure}, + success_status_code=httplib.OK) + + def download_object_as_stream(self, obj, chunk_size=None): + container_name = obj.container.name + object_name = obj.name + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='GET', raw=True) + + return self._get_object(obj=obj, callback=read_in_chunks, + response=response, + callback_kwargs={ 'iterator': response.response, + 'chunk_size': chunk_size}, + success_status_code=httplib.OK) + + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True): + """ + Upload an object. + + Note: This will override file with a same name if it already exists. + """ + upload_func = self._upload_file + upload_func_kwargs = { 'file_path': file_path } + + return self._put_object(container=container, object_name=object_name, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + extra=extra, file_path=file_path, + verify_hash=verify_hash) + + def upload_object_via_stream(self, iterator, + container, object_name, extra=None): + if isinstance(iterator, file): + iterator = iter(iterator) + + upload_func = self._stream_data + upload_func_kwargs = { 'iterator': iterator } + + return self._put_object(container=container, object_name=object_name, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + extra=extra, iterator=iterator) + + def delete_object(self, obj): + container_name = self._clean_container_name(obj.container.name) + object_name = self._clean_object_name(obj.name) + + response = self.connection.request( + '/%s/%s' % (container_name, object_name), method='DELETE') + + if response.status == httplib.NO_CONTENT: + return True + elif response.status == httplib.NOT_FOUND: + raise ObjectDoesNotExistError(value='', object_name=object_name, + driver=self) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def ex_get_meta_data(self): + response = self.connection.request('', method='HEAD') + + if response.status == httplib.NO_CONTENT: + container_count = response.headers.get( + 'x-account-container-count', 'unknown') + object_count = response.headers.get( + 'x-account-object-count', 'unknown') + bytes_used = response.headers.get( + 'x-account-bytes-used', 'unknown') + + return { 'container_count': int(container_count), + 'object_count': int(object_count), + 'bytes_used': int(bytes_used) } + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def _put_object(self, container, object_name, upload_func, + upload_func_kwargs, extra=None, file_path=None, + iterator=None, verify_hash=True): + extra = extra or {} + container_name_cleaned = self._clean_container_name(container.name) + object_name_cleaned = self._clean_object_name(object_name) + content_type = extra.get('content_type', None) + meta_data = extra.get('meta_data', None) + + headers = {} + if meta_data: + for key, value in meta_data.iteritems(): + key = 'X-Object-Meta-%s' % (key) + headers[key] = value + + request_path = '/%s/%s' % (container_name_cleaned, object_name_cleaned) + result_dict = self._upload_object(object_name=object_name, + content_type=content_type, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + request_path=request_path, + request_method='PUT', + headers=headers, file_path=file_path, + iterator=iterator) + + response = result_dict['response'].response + bytes_transferred = result_dict['bytes_transferred'] + server_hash = result_dict['response'].headers.get('etag', None) + + if response.status == httplib.EXPECTATION_FAILED: + raise LibcloudError(value='Missing content-type header', + driver=self) + elif verify_hash and not server_hash: + raise LibcloudError(value='Server didn\'t return etag', + driver=self) + elif (verify_hash and result_dict['data_hash'] != server_hash): + raise ObjectHashMismatchError( + value=('MD5 hash checksum does not match (expected=%s, ' + + 'actual=%s)') % (result_dict['data_hash'], server_hash), + object_name=object_name, driver=self) + elif response.status == httplib.CREATED: + obj = Object( + name=object_name, size=bytes_transferred, hash=server_hash, + extra=None, meta_data=meta_data, container=container, + driver=self) + + return obj + else: + # @TODO: Add test case for this condition (probably 411) + raise LibcloudError('status_code=%s' % (response.status), + driver=self) + + def _clean_container_name(self, name): + """ + Clean container name. + """ + if name.startswith('/'): + name = name[1:] + name = urllib.quote(name) + + if name.find('/') != -1: + raise InvalidContainerNameError(value='Container name cannot' + ' contain slashes', + container_name=name, driver=self) + + if len(name) > 256: + raise InvalidContainerNameError(value='Container name cannot be' + ' longer than 256 bytes', + container_name=name, driver=self) + + + return name + + def _clean_object_name(self, name): + name = urllib.quote(name) + return name + + def _to_container_list(self, response): + # @TODO: Handle more then 10k containers - use "lazy list"? + containers = [] + + for container in response: + extra = { 'object_count': int(container['count']), + 'size': int(container['bytes'])} + containers.append(Container(name=container['name'], extra=extra, + driver=self)) + + return containers + + def _to_object_list(self, response, container): + objects = [] + + for obj in response: + name = obj['name'] + size = int(obj['bytes']) + hash = obj['hash'] + extra = { 'content_type': obj['content_type'], + 'last_modified': obj['last_modified'] } + objects.append(Object( + name=name, size=size, hash=hash, extra=extra, + meta_data=None, container=container, driver=self)) + + return objects + + def _headers_to_container(self, name, headers): + size = int(headers.get('x-container-bytes-used', 0)) + object_count = int(headers.get('x-container-object-count', 0)) + + extra = { 'object_count': object_count, + 'size': size } + container = Container(name=name, extra=extra, driver=self) + return container + + def _headers_to_object(self, name, container, headers): + size = int(headers.pop('content-length', 0)) + last_modified = headers.pop('last-modified', None) + etag = headers.pop('etag', None) + content_type = headers.pop('content-type', None) + + meta_data = {} + for key, value in headers.iteritems(): + if key.find('x-object-meta-') != -1: + key = key.replace('x-object-meta-', '') + meta_data[key] = value + + extra = { 'content_type': content_type, 'last_modified': last_modified } + + obj = Object(name=name, size=size, hash=etag, extra=extra, + meta_data=meta_data, container=container, driver=self) + return obj + + +class SwiftOSStorageDriver(SwiftStorageDriver): + """ + Cloudfiles storage driver for the US endpoint. + """ + + type = Provider.SWIFT_OS + name = 'Swift (US)' + connectionCls = SwiftOSConnection + diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index 43d9b32775..1102b93d1e 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -23,6 +23,8 @@ ('libcloud.storage.drivers.cloudfiles', 'CloudFilesUSStorageDriver'), Provider.CLOUDFILES_UK: ('libcloud.storage.drivers.cloudfiles', 'CloudFilesUKStorageDriver'), + Provider.SWIFT: + ('libcloud.storage.drivers.swift', 'SwiftOSStorageDriver'), Provider.S3: ('libcloud.storage.drivers.s3', 'S3StorageDriver'), Provider.S3_US_WEST: diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 8aa1dc0df5..54dd7d0e2b 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -37,6 +37,7 @@ class Provider(object): @cvar S3_EU_WEST: Amazon S3 EU West (Ireland) @cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore) @cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo) + @cvar SWIFT: Swfit provider allowing the usage of OpenStack Swift """ DUMMY = 0 CLOUDFILES_US = 1 @@ -46,6 +47,7 @@ class Provider(object): S3_EU_WEST = 5 S3_AP_SOUTHEAST = 6 S3_AP_NORTHEAST = 7 + SWIFT = 8 class ContainerError(LibcloudError): error_type = 'ContainerError' @@ -55,9 +57,9 @@ def __init__(self, value, driver, container_name): super(ContainerError, self).__init__(value=value, driver=driver) def __str__(self): - return ('<%s in %s, container = %s>' % + return ('<%s in %s, container=%s, value=%s>' % (self.error_type, repr(self.driver), - self.container_name)) + self.container_name, self.value)) class ObjectError(LibcloudError): error_type = 'ContainerError' @@ -67,8 +69,8 @@ def __init__(self, value, driver, object_name): super(ObjectError, self).__init__(value=value, driver=driver) def __str__(self): - return '<%s in %s, object = %s>' % (self.error_type, repr(self.driver), - self.object_name) + return '<%s in %s, value=%s, object = %s>' % (self.error_type, repr(self.driver), + self.value, self.object_name) class ContainerAlreadyExistsError(ContainerError): error_type = 'ContainerAlreadyExistsError' diff --git a/libcloud/utils.py b/libcloud/utils.py index df2412ffdd..a9ab20beff 100644 --- a/libcloud/utils.py +++ b/libcloud/utils.py @@ -176,6 +176,9 @@ def findattr(element, xpath, namespace): def findall(element, xpath, namespace): return element.findall(fixxpath(xpath=xpath, namespace=namespace)) +def reverse_dict(dictionary): + return dict([ (value, key) for key, value in dictionary.iteritems() ]) + def get_driver(drivers, provider): """ Get a driver. diff --git a/setup.py b/setup.py index 5878a35d67..d4f1273aea 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,18 @@ HTML_VIEWSOURCE_BASE = 'https://svn.apache.org/viewvc/incubator/libcloud/trunk' PROJECT_BASE_DIR = 'http://incubator.apache.org/libcloud/' -TEST_PATHS = [ 'test', 'test/compute', 'test/storage' ] +TEST_PATHS = [ 'test', 'test/compute', 'test/storage' , 'test/loadbalancer'] DOC_TEST_MODULES = [ 'libcloud.compute.drivers.dummy', 'libcloud.storage.drivers.dummy' ] +def read_version_string(): + version = None + sys.path.insert(0, pjoin(os.getcwd())) + from libcloud import __version__ + version = __version__ + sys.path.pop(0) + return version + class TestCommand(Command): user_options = [] @@ -139,7 +147,7 @@ def run(self): setup( name='apache-libcloud', - version='0.4.3', + version=read_version_string(), description='A unified interface into many cloud server providers', author='Apache Software Foundation', author_email='libcloud@incubator.apache.org', @@ -149,17 +157,17 @@ def run(self): 'libcloud.common', 'libcloud.compute', 'libcloud.compute.drivers', + 'libcloud.storage', + 'libcloud.storage.drivers', 'libcloud.drivers', - 'libcloud.resource', - 'libcloud.resource.lb', - 'libcloud.resource.lb.drivers', + 'libcloud.loadbalancer', + 'libcloud.loadbalancer.drivers', ], package_dir={ 'libcloud': 'libcloud', - 'libcloud.drivers': 'libcloud/drivers' }, package_data={ - 'libcloud': ['data/*.json'], + 'libcloud': ['data/*.json'] }, license='Apache License (2.0)', url='http://incubator.apache.org/libcloud/', diff --git a/test/__init__.py b/test/__init__.py index 040a21790a..1ec9a52bde 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -12,13 +12,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import httplib import random +import unittest from cStringIO import StringIO from urllib2 import urlparse from cgi import parse_qs +class LibcloudTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + self._visited_urls = [] + self._executed_mock_methods = [] + super(LibcloudTestCase, self).__init__(*args, **kwargs) + + def setUp(self): + self._visited_urls = [] + self._executed_mock_methods = [] + + def _add_visited_url(self, url): + self._visited_urls.append(url) + + def _add_executed_mock_method(self, method_name): + self._executed_mock_methods.append(method_name) + + def assertExecutedMethodCount(self, expected): + print self._executed_mock_methods + actual = len(self._executed_mock_methods) + self.assertEqual(actual, expected, + 'expected %d, but %d mock methods were executed' + % (expected, actual)) + class multipleresponse(object): """ A decorator that allows MockHttp objects to return multi responses @@ -112,6 +137,8 @@ class MockHttp(BaseMockHttpObject): type = None use_param = None # will use this param to namespace the request function + test = None # TestCase instance which is using this mock + def __init__(self, host, port, *args, **kwargs): self.host = host self.port = port @@ -127,6 +154,11 @@ def request(self, method, url, body=None, headers=None, raw=False): use_param=self.use_param, qs=qs, path=path) meth = getattr(self, meth_name) + + if self.test and isinstance(self.test, LibcloudTestCase): + self.test._add_visited_url(url=url) + self.test._add_executed_mock_method(method_name=meth_name) + status, body, headers, reason = meth(method, url, body, headers) self.response = self.responseCls(status, body, headers, reason) @@ -154,6 +186,18 @@ def _example_fail(self, method, url, body, headers): return (httplib.FORBIDDEN, 'Oh Noes!', {'X-Foo': 'fail'}, httplib.responses[httplib.FORBIDDEN]) +class MockHttpTestCase(MockHttp, unittest.TestCase): + # Same as the MockHttp class, but you can also use assertions in the + # classes which inherit from this one. + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self) + + if kwargs.get('host', None) and kwargs.get('port', None): + MockHttp.__init__(self, *args, **kwargs) + + def runTest(self): + pass + class StorageMockHttp(MockHttp): def putrequest(self, method, action): pass @@ -167,7 +211,6 @@ def endheaders(self): def send(self, data): pass - class MockRawResponse(BaseMockHttpObject): """ Mock RawResponse object suitable for testing. diff --git a/test/compute/test_ec2.py b/test/compute/test_ec2.py index 1a38ae4767..3f6754b59b 100644 --- a/test/compute/test_ec2.py +++ b/test/compute/test_ec2.py @@ -21,15 +21,16 @@ from libcloud.compute.drivers.ec2 import EC2APNENodeDriver, IdempotentParamError from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation -from test import MockHttp +from test import MockHttp, LibcloudTestCase from test.compute import TestCaseMixin from test.file_fixtures import ComputeFileFixtures from test.secrets import EC2_ACCESS_ID, EC2_SECRET -class EC2Tests(unittest.TestCase, TestCaseMixin): +class EC2Tests(LibcloudTestCase, TestCaseMixin): def setUp(self): + EC2MockHttp.test = self EC2NodeDriver.connectionCls.conn_classes = (None, EC2MockHttp) EC2MockHttp.use_param = 'Action' EC2MockHttp.type = None @@ -335,10 +336,18 @@ def test_list_sizes(self): def test_list_nodes(self): # overridden from EC2Tests -- Nimbus doesn't support elastic IPs. node = self.driver.list_nodes()[0] + self.assertExecutedMethodCount(0) public_ips = node.public_ip self.assertEqual(node.id, 'i-4382922a') self.assertEqual(len(node.public_ip), 1) self.assertEqual(public_ips[0], '1.2.3.5') + def test_ex_create_tags(self): + # Nimbus doesn't support creating tags so this one should be a + # passthrough + node = self.driver.list_nodes()[0] + self.driver.ex_create_tags(node=node, tags={'foo': 'bar'}) + self.assertExecutedMethodCount(0) + if __name__ == '__main__': sys.exit(unittest.main()) diff --git a/test/compute/test_elastichosts.py b/test/compute/test_elastichosts.py index 5efb9fd665..a22abe4259 100644 --- a/test/compute/test_elastichosts.py +++ b/test/compute/test_elastichosts.py @@ -25,7 +25,6 @@ from libcloud.common.types import InvalidCredsError, MalformedResponseError from test import MockHttp -from test.compute import TestCaseMixin from test.file_fixtures import ComputeFileFixtures class ElasticHostsTestCase(unittest.TestCase): diff --git a/test/compute/test_gandi.py b/test/compute/test_gandi.py index 00e13225d8..25bf95594a 100644 --- a/test/compute/test_gandi.py +++ b/test/compute/test_gandi.py @@ -36,7 +36,12 @@ def request(self, host, handler, request_body, verbose=0): mock = GandiMockHttp(host, 80) mock.request('POST', "%s/%s" % (handler, method)) resp = mock.getresponse() - return self._parse_response(resp.body, None) + + if sys.version[0] == '2' and sys.version[2] == '7': + response = self.parse_response(resp) + else: + response = self.parse_response(resp.body) + return response class GandiTests(unittest.TestCase): diff --git a/test/compute/test_gogrid.py b/test/compute/test_gogrid.py index e807befb17..ddeffe4b4f 100644 --- a/test/compute/test_gogrid.py +++ b/test/compute/test_gogrid.py @@ -21,7 +21,7 @@ from libcloud.common.types import LibcloudError, InvalidCredsError from libcloud.common.gogrid import GoGridIpAddress from libcloud.compute.drivers.gogrid import GoGridNodeDriver -from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation +from libcloud.compute.base import Node, NodeImage, NodeSize from test import MockHttp # pylint: disable-msg=E0611 from test.compute import TestCaseMixin # pylint: disable-msg=E0611 @@ -69,7 +69,7 @@ def test_reboot_node_not_successful(self): node = Node(90967, None, None, None, None, self.driver) try: - ret = self.driver.reboot_node(node) + self.driver.reboot_node(node) except Exception: pass else: diff --git a/test/compute/test_rackspace.py b/test/compute/test_rackspace.py index b46f3ddf9a..dbe92ee577 100644 --- a/test/compute/test_rackspace.py +++ b/test/compute/test_rackspace.py @@ -16,7 +16,7 @@ import unittest import httplib -from libcloud.common.types import InvalidCredsError +from libcloud.common.types import InvalidCredsError, MalformedResponseError from libcloud.compute.drivers.rackspace import RackspaceNodeDriver as Rackspace from libcloud.compute.base import Node, NodeImage, NodeSize @@ -46,8 +46,17 @@ def test_auth_missing_key(self): RackspaceMockHttp.type = 'UNAUTHORIZED_MISSING_KEY' try: self.driver = Rackspace(RACKSPACE_USER, RACKSPACE_KEY) - except InvalidCredsError, e: - self.assertEqual(True, isinstance(e, InvalidCredsError)) + except MalformedResponseError, e: + self.assertEqual(True, isinstance(e, MalformedResponseError)) + else: + self.fail('test should have thrown') + + def test_auth_server_error(self): + RackspaceMockHttp.type = 'INTERNAL_SERVER_ERROR' + try: + self.driver = Rackspace(RACKSPACE_USER, RACKSPACE_KEY) + except MalformedResponseError, e: + self.assertEqual(True, isinstance(e, MalformedResponseError)) else: self.fail('test should have thrown') @@ -197,6 +206,9 @@ def _v1_0(self, method, url, body, headers): def _v1_0_UNAUTHORIZED(self, method, url, body, headers): return (httplib.UNAUTHORIZED, "", {}, httplib.responses[httplib.UNAUTHORIZED]) + def _v1_0_INTERNAL_SERVER_ERROR(self, method, url, body, headers): + return (httplib.INTERNAL_SERVER_ERROR, "

500: Internal Server Error

", {}, httplib.responses[httplib.INTERNAL_SERVER_ERROR]) + def _v1_0_UNAUTHORIZED_MISSING_KEY(self, method, url, body, headers): headers = {'x-server-management-url': 'https://servers.api.rackspacecloud.com/v1.0/slug', 'x-auth-token': 'FE011C19-CF86-4F87-BE5D-9229145D7A06', @@ -277,7 +289,5 @@ def _v1_0_slug_shared_ip_groups_detail(self, method, url, body, headers): def _v1_0_slug_servers_3445_ips_public_67_23_21_133(self, method, url, body, headers): return (httplib.ACCEPTED, "", {}, httplib.responses[httplib.ACCEPTED]) - - if __name__ == '__main__': sys.exit(unittest.main()) diff --git a/test/file_fixtures.py b/test/file_fixtures.py index 01ee3c76ab..578b139b03 100644 --- a/test/file_fixtures.py +++ b/test/file_fixtures.py @@ -20,7 +20,7 @@ FIXTURES_ROOT = { 'compute': 'compute/fixtures', 'storage': 'storage/fixtures', - 'resource': 'resource/fixtures', + 'loadbalancer': 'loadbalancer/fixtures', } class FileFixtures(object): @@ -46,7 +46,7 @@ def __init__(self, sub_dir=''): super(StorageFileFixtures, self).__init__(fixtures_type='storage', sub_dir=sub_dir) -class ResourceFileFixtures(FileFixtures): +class LoadBalancerFileFixtures(FileFixtures): def __init__(self, sub_dir=''): - super(ResourceFileFixtures, self).__init__(fixtures_type='resource', + super(LoadBalancerFileFixtures, self).__init__(fixtures_type='loadbalancer', sub_dir=sub_dir) diff --git a/libcloud/resource/__init__.py b/test/loadbalancer/__init__.py similarity index 96% rename from libcloud/resource/__init__.py rename to test/loadbalancer/__init__.py index 63ed5734fe..ae1e83eeb3 100644 --- a/libcloud/resource/__init__.py +++ b/test/loadbalancer/__init__.py @@ -12,8 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -__all__ = [ - 'lb' -] diff --git a/test/resource/fixtures/lb/gogrid/ip_list.json b/test/loadbalancer/fixtures/gogrid/ip_list.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/ip_list.json rename to test/loadbalancer/fixtures/gogrid/ip_list.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_add.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_add.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_add.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_add.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_edit.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_edit.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_edit.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_edit.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_get.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_get.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_get.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_get.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_list.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_list.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_list.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_list.json diff --git a/test/loadbalancer/fixtures/gogrid/unexpected_error.json b/test/loadbalancer/fixtures/gogrid/unexpected_error.json new file mode 100644 index 0000000000..87ed4e56ba --- /dev/null +++ b/test/loadbalancer/fixtures/gogrid/unexpected_error.json @@ -0,0 +1 @@ +{"summary":{"total":1,"start":0,"returned":1},"status":"failure","method":"/grid/loadbalancer/add","list":[{"message":"An unexpected server error has occured. Please email this error to apisupport@gogrid.com. Error Message : null","object":"error","errorcode":"UnexpectedException"}]} diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes_post.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes_post.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes_post.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes_post.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_post.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_post.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_post.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_post.json diff --git a/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json new file mode 100644 index 0000000000..d966c396b7 --- /dev/null +++ b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json @@ -0,0 +1,43 @@ +{"protocols": [ + { + "name": "HTTP", + "port": "80" + }, + { + "name": "FTP", + "port": "21" + }, + { + "name": "IMAPv4", + "port": "143" + }, + { + "name": "POP3", + "port": "110" + }, + { + "name": "SMTP", + "port": "25" + }, + { + "name": "LDAP", + "port": "389" + }, + { + "name": "HTTPS", + "port": "443" + }, + { + "name": "IMAPS", + "port": "993" + }, + { + "name": "POP3S", + "port": "995" + }, + { + "name": "LDAPS", + "port": "636" + } + ] +} diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py new file mode 100644 index 0000000000..5db7ba9351 --- /dev/null +++ b/test/loadbalancer/test_gogrid.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import httplib +import sys +import unittest +from urlparse import urlparse, parse_qsl + +from libcloud.common.types import LibcloudError +from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm +from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver + +from test import MockHttpTestCase +from test.file_fixtures import LoadBalancerFileFixtures + +class GoGridTests(unittest.TestCase): + + def setUp(self): + GoGridLBDriver.connectionCls.conn_classes = (None, + GoGridLBMockHttp) + GoGridLBMockHttp.type = None + self.driver = GoGridLBDriver('user', 'key') + + def test_list_protocols(self): + protocols = self.driver.list_protocols() + + self.assertEqual(len(protocols), 1) + self.assertEqual(protocols[0], 'http') + + def test_list_balancers(self): + balancers = self.driver.list_balancers() + + self.assertEquals(len(balancers), 2) + self.assertEquals(balancers[0].name, "foo") + self.assertEquals(balancers[0].id, "23517") + self.assertEquals(balancers[1].name, "bar") + self.assertEquals(balancers[1].id, "23526") + + def test_create_balancer(self): + balancer = self.driver.create_balancer(name='test2', + port=80, + protocol='http', + algorithm=Algorithm.ROUND_ROBIN, + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) + ) + + self.assertEquals(balancer.name, 'test2') + self.assertEquals(balancer.id, '123') + + def test_create_balancer_UNEXPECTED_ERROR(self): + # Try to create new balancer and attach members with an IP address which + # does not belong to this account + GoGridLBMockHttp.type = 'UNEXPECTED_ERROR' + + try: + self.driver.create_balancer(name='test2', + port=80, + protocol='http', + algorithm=Algorithm.ROUND_ROBIN, + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) + ) + except LibcloudError, e: + self.assertTrue(str(e).find('tried to add a member with an IP address not assigned to your account') != -1) + else: + self.fail('Exception was not thrown') + + def test_destroy_balancer(self): + balancer = self.driver.list_balancers()[0] + + ret = self.driver.destroy_balancer(balancer) + self.assertTrue(ret) + + def test_get_balancer(self): + balancer = self.driver.get_balancer(balancer_id='23530') + + self.assertEquals(balancer.name, 'test2') + self.assertEquals(balancer.id, '23530') + + def test_balancer_list_members(self): + balancer = self.driver.get_balancer(balancer_id='23530') + members = balancer.list_members() + + expected_members = set([u'10.0.0.78:80', u'10.0.0.77:80', + u'10.0.0.76:80']) + + self.assertEquals(len(members), 3) + self.assertEquals(expected_members, + set(["%s:%s" % (member.ip, member.port) for member in members])) + + def test_balancer_attach_member(self): + balancer = LoadBalancer(23530, None, None, None, None, None) + member = self.driver.balancer_attach_member(balancer, + Member(None, ip='10.0.0.75', port='80')) + + self.assertEquals(member.ip, '10.0.0.75') + self.assertEquals(member.port, 80) + + def test_balancer_detach_member(self): + balancer = LoadBalancer(23530, None, None, None, None, None) + member = self.driver.balancer_list_members(balancer)[0] + + ret = self.driver.balancer_detach_member(balancer, member) + + self.assertTrue(ret) + +class GoGridLBMockHttp(MockHttpTestCase): + fixtures = LoadBalancerFileFixtures('gogrid') + + def _api_grid_loadbalancer_list(self, method, url, body, headers): + body = self.fixtures.load('loadbalancer_list.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_ip_list(self, method, url, body, headers): + body = self.fixtures.load('ip_list.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_loadbalancer_add(self, method, url, body, headers): + qs = dict(parse_qsl(urlparse(url).query)) + self.assertEqual(qs['loadbalancer.type'], 'round robin') + + body = self.fixtures.load('loadbalancer_add.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_ip_list_UNEXPECTED_ERROR(self, method, url, body, headers): + return self._api_grid_ip_list(method, url, body, headers) + + def _api_grid_loadbalancer_add_UNEXPECTED_ERROR(self, method, url, body, headers): + body = self.fixtures.load('unexpected_error.json') + return (httplib.INTERNAL_SERVER_ERROR, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_loadbalancer_delete(self, method, url, body, headers): + body = self.fixtures.load('loadbalancer_add.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_loadbalancer_get(self, method, url, body, headers): + body = self.fixtures.load('loadbalancer_get.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + def _api_grid_loadbalancer_edit(self, method, url, body, headers): + body = self.fixtures.load('loadbalancer_edit.json') + return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + + +if __name__ == "__main__": + sys.exit(unittest.main()) diff --git a/test/resource/lb/test_rackspace.py b/test/loadbalancer/test_rackspace.py similarity index 56% rename from test/resource/lb/test_rackspace.py rename to test/loadbalancer/test_rackspace.py index 91d6cd263a..0ffdf9de98 100644 --- a/test/resource/lb/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -1,13 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import httplib -import os.path import sys import unittest -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.drivers.rackspace import RackspaceLBDriver +try: + import json +except ImportError: + import simplejson as json + +from libcloud.loadbalancer.base import Member, Algorithm +from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver -from test import MockHttp, MockRawResponse -from test.file_fixtures import ResourceFileFixtures +from test import MockHttp +from test.file_fixtures import LoadBalancerFileFixtures class RackspaceLBTests(unittest.TestCase): @@ -17,6 +36,12 @@ def setUp(self): RackspaceLBMockHttp.type = None self.driver = RackspaceLBDriver('user', 'key') + def test_list_protocols(self): + protocols = self.driver.list_protocols() + + self.assertEqual(len(protocols), 10) + self.assertTrue('http' in protocols) + def test_list_balancers(self): balancers = self.driver.list_balancers() @@ -29,8 +54,9 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, - nodes=(LBNode(None, '10.1.0.10', 80), - LBNode(None, '10.1.0.11', 80)) + algorithm=Algorithm.ROUND_ROBIN, + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') @@ -42,37 +68,37 @@ def test_destroy_balancer(self): ret = self.driver.destroy_balancer(balancer) self.assertTrue(ret) - def test_balancer_detail(self): - balancer = self.driver.balancer_detail(balancer_id='8290') + def test_get_balancer(self): + balancer = self.driver.get_balancer(balancer_id='8290') self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '8290') - def test_balancer_list_nodes(self): - balancer = self.driver.balancer_detail(balancer_id='8290') - nodes = balancer.list_nodes() + def test_balancer_list_members(self): + balancer = self.driver.get_balancer(balancer_id='8290') + members = balancer.list_members() - self.assertEquals(len(nodes), 2) + self.assertEquals(len(members), 2) self.assertEquals(set(['10.1.0.10:80', '10.1.0.11:80']), - set(["%s:%s" % (node.ip, node.port) for node in nodes])) + set(["%s:%s" % (member.ip, member.port) for member in members])) - def test_balancer_attach_node(self): - balancer = self.driver.balancer_detail(balancer_id='8290') - node = balancer.attach_node(ip='10.1.0.12', port='80') + def test_balancer_attach_member(self): + balancer = self.driver.get_balancer(balancer_id='8290') + member = balancer.attach_member(Member(None, ip='10.1.0.12', port='80')) - self.assertEquals(node.ip, '10.1.0.12') - self.assertEquals(node.port, 80) + self.assertEquals(member.ip, '10.1.0.12') + self.assertEquals(member.port, 80) - def test_balancer_detach_node(self): - balancer = self.driver.balancer_detail(balancer_id='8290') - node = balancer.list_nodes()[0] + def test_balancer_detach_member(self): + balancer = self.driver.get_balancer(balancer_id='8290') + member = balancer.list_members()[0] - ret = balancer.detach_node(node) + ret = balancer.detach_member(member) self.assertTrue(ret) -class RackspaceLBMockHttp(MockHttp): - fixtures = ResourceFileFixtures(os.path.join('lb', 'rackspace')) +class RackspaceLBMockHttp(MockHttp, unittest.TestCase): + fixtures = LoadBalancerFileFixtures('rackspace') def _v1_0(self, method, url, body, headers): headers = {'x-server-management-url': 'https://servers.api.rackspacecloud.com/v1.0/slug', @@ -82,11 +108,20 @@ def _v1_0(self, method, url, body, headers): 'x-storage-url': 'https://storage4.clouddrive.com/v1/MossoCloudFS_FE011C19-CF86-4F87-BE5D-9229145D7A06'} return (httplib.NO_CONTENT, "", headers, httplib.responses[httplib.NO_CONTENT]) + def _v1_0_slug_loadbalancers_protocols(self, method, url, body, headers): + body = self.fixtures.load('v1_slug_loadbalancers_protocols.json') + return (httplib.ACCEPTED, body, {}, + httplib.responses[httplib.ACCEPTED]) + def _v1_0_slug_loadbalancers(self, method, url, body, headers): if method == "GET": body = self.fixtures.load('v1_slug_loadbalancers.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) elif method == "POST": + body_json = json.loads(body) + self.assertEqual(body_json['loadBalancer']['protocol'], 'HTTP') + self.assertEqual(body_json['loadBalancer']['algorithm'], 'ROUND_ROBIN') + body = self.fixtures.load('v1_slug_loadbalancers_post.json') return (httplib.ACCEPTED, body, {}, httplib.responses[httplib.ACCEPTED]) diff --git a/test/resource/lb/test_gogrid.py b/test/resource/lb/test_gogrid.py deleted file mode 100644 index af94c63bc4..0000000000 --- a/test/resource/lb/test_gogrid.py +++ /dev/null @@ -1,107 +0,0 @@ -import httplib -import os.path -import sys -import unittest - -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.drivers.gogrid import GoGridLBDriver - -from test import MockHttp, MockRawResponse -from test.file_fixtures import ResourceFileFixtures - -class GoGridTests(unittest.TestCase): - - def setUp(self): - GoGridLBDriver.connectionCls.conn_classes = (None, - GoGridLBMockHttp) - GoGridLBMockHttp.type = None - self.driver = GoGridLBDriver('user', 'key') - - def test_list_balancers(self): - balancers = self.driver.list_balancers() - - self.assertEquals(len(balancers), 2) - self.assertEquals(balancers[0].name, "foo") - self.assertEquals(balancers[0].id, "23517") - self.assertEquals(balancers[1].name, "bar") - self.assertEquals(balancers[1].id, "23526") - - def test_create_balancer(self): - balancer = self.driver.create_balancer(name='test2', - port=80, - nodes=(LBNode(None, '10.1.0.10', 80), - LBNode(None, '10.1.0.11', 80)) - ) - - self.assertEquals(balancer.name, 'test2') - self.assertEquals(balancer.id, '123') - - def test_destroy_balancer(self): - balancer = self.driver.list_balancers()[0] - - ret = self.driver.destroy_balancer(balancer) - self.assertTrue(ret) - - def test_balancer_detail(self): - balancer = self.driver.balancer_detail(balancer_id='23530') - - self.assertEquals(balancer.name, 'test2') - self.assertEquals(balancer.id, '23530') - - def test_balancer_list_nodes(self): - balancer = self.driver.balancer_detail(balancer_id='23530') - nodes = balancer.list_nodes() - - expected_nodes = set([u'10.0.0.78:80', u'10.0.0.77:80', - u'10.0.0.76:80']) - - self.assertEquals(len(nodes), 3) - self.assertEquals(expected_nodes, - set(["%s:%s" % (node.ip, node.port) for node in nodes])) - - def test_balancer_attach_node(self): - balancer = LB(23530, None, None, None, None, None) - node = self.driver.balancer_attach_node(balancer, - ip='10.0.0.75', port='80') - - self.assertEquals(node.ip, '10.0.0.75') - self.assertEquals(node.port, 80) - - def test_balancer_detach_node(self): - balancer = LB(23530, None, None, None, None, None) - node = self.driver.balancer_list_nodes(balancer)[0] - - ret = self.driver.balancer_detach_node(balancer, node) - - self.assertTrue(ret) - -class GoGridLBMockHttp(MockHttp): - fixtures = ResourceFileFixtures(os.path.join('lb', 'gogrid')) - - def _api_grid_loadbalancer_list(self, method, url, body, headers): - body = self.fixtures.load('loadbalancer_list.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - def _api_grid_ip_list(self, method, url, body, headers): - body = self.fixtures.load('ip_list.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - def _api_grid_loadbalancer_add(self, method, url, body, headers): - body = self.fixtures.load('loadbalancer_add.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - def _api_grid_loadbalancer_delete(self, method, url, body, headers): - body = self.fixtures.load('loadbalancer_add.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - def _api_grid_loadbalancer_get(self, method, url, body, headers): - body = self.fixtures.load('loadbalancer_get.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - def _api_grid_loadbalancer_edit(self, method, url, body, headers): - body = self.fixtures.load('loadbalancer_edit.json') - return (httplib.OK, body, {}, httplib.responses[httplib.OK]) - - -if __name__ == "__main__": - sys.exit(unittest.main()) diff --git a/test/storage/fixtures/swift/list_container_objects.json b/test/storage/fixtures/swift/list_container_objects.json new file mode 100644 index 0000000000..4c47200bb4 --- /dev/null +++ b/test/storage/fixtures/swift/list_container_objects.json @@ -0,0 +1,14 @@ +[ + {"name":"foo test 1","hash":"16265549b5bda64ecdaa5156de4c97cc", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo test 2","hash":"16265549b5bda64ecdaa5156de4c97bb", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo tes 3","hash":"16265549b5bda64ecdaa5156de4c97ee", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:46.549890"}, + {"name":"foo test 3","hash":"16265549b5bda64ecdaa5156de4c97ff", + "bytes":1160520,"content_type":"application/text", + "last_modified":"2011-01-25T22:01:50.351810"} +] diff --git a/test/storage/fixtures/swift/list_container_objects_empty.json b/test/storage/fixtures/swift/list_container_objects_empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/storage/fixtures/swift/list_container_objects_empty.json @@ -0,0 +1 @@ +{} diff --git a/test/storage/fixtures/swift/list_containers.json b/test/storage/fixtures/swift/list_containers.json new file mode 100644 index 0000000000..ded31c02c3 --- /dev/null +++ b/test/storage/fixtures/swift/list_containers.json @@ -0,0 +1,5 @@ +[ + {"name":"container1","count":4,"bytes":3484450}, + {"name":"container2","count":120,"bytes":340084450}, + {"name":"container3","count":0,"bytes":0} +] diff --git a/test/storage/fixtures/swift/list_containers_empty.json b/test/storage/fixtures/swift/list_containers_empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/storage/fixtures/swift/list_containers_empty.json @@ -0,0 +1 @@ +{} diff --git a/test/storage/fixtures/swift/meta_data.json b/test/storage/fixtures/swift/meta_data.json new file mode 100644 index 0000000000..5049f58492 --- /dev/null +++ b/test/storage/fixtures/swift/meta_data.json @@ -0,0 +1 @@ +{"bytes_used": 1234567, "container_count": 10, "object_count": 400} diff --git a/test/storage/test_cloudfiles.py b/test/storage/test_cloudfiles.py index f7d0ea80e5..a79a86b2cb 100644 --- a/test/storage/test_cloudfiles.py +++ b/test/storage/test_cloudfiles.py @@ -35,6 +35,8 @@ from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 +current_hash = None + class CloudFilesTests(unittest.TestCase): def setUp(self): @@ -50,9 +52,6 @@ def setUp(self): def tearDown(self): self._remove_test_file() - def test_get_meta_data(self): - self.driver.get_meta_data() - def test_invalid_json_throws_exception(self): CloudFilesMockHttp.type = 'MALFORMED_JSON' try: @@ -110,8 +109,8 @@ def test_get_object_success(self): object_name='test_object') self.assertEqual(obj.container.name, 'test_container') self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') self.assertEqual(obj.extra['content_type'], 'application/zip') - self.assertEqual(obj.extra['etag'], '6b21c4a111ac178feacf9ec9d0c71f17') self.assertEqual( obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') self.assertEqual(obj.meta_data['foo-bar'], 'test 1') @@ -279,7 +278,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='footest123') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -397,6 +396,12 @@ def test_delete_object_not_found(self): else: self.fail('Object does not exist but an exception was not thrown') + def test_ex_get_meta_data(self): + meta_data = self.driver.ex_get_meta_data() + self.assertTrue(isinstance(meta_data, dict)) + self.assertTrue('object_count' in meta_data) + self.assertTrue('container_count' in meta_data) + self.assertTrue('bytes_used' in meta_data) def _remove_test_file(self): file_path = os.path.abspath(__file__) + '.temp' @@ -598,16 +603,19 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( # test_object_upload_success body = '' - headers = copy.deepcopy(self.base_headers) - headers.update(headers) + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'hash343hhash89h932439jsaa89' return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( self, method, url, body, headers): # test_object_upload_invalid_hash body = '' - headers = self.base_headers - return (httplib.UNPROCESSABLE_ENTITY, body, headers, + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'foobar' + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( @@ -641,10 +649,13 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data( self, method, url, body, headers): # test_upload_object_via_stream_success + headers = {} + headers.update(self.base_headers) + headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e' body = 'test' return (httplib.CREATED, body, - self.base_headers, + headers, httplib.responses[httplib.OK]) if __name__ == '__main__': diff --git a/test/storage/test_s3.py b/test/storage/test_s3.py index 5395daf4b5..8e31c00e24 100644 --- a/test/storage/test_s3.py +++ b/test/storage/test_s3.py @@ -73,14 +73,6 @@ def test_bucket_is_located_in_different_region(self): else: self.fail('Exception was not thrown') - def test_get_meta_data(self): - try: - self.driver.get_meta_data() - except NotImplementedError: - pass - else: - self.fail('Exception was not thrown') - def test_list_containers_empty(self): S3MockHttp.type = 'list_containers_EMPTY' containers = self.driver.list_containers() @@ -277,7 +269,7 @@ def test_upload_object_invalid_ex_storage_class(self): try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661', + verify_hash=True, ex_storage_class='invalid-class') except ValueError, e: self.assertTrue(str(e).lower().find('invalid storage class') != -1) @@ -302,7 +294,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -328,7 +320,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -351,7 +343,7 @@ def upload_file(self, response, file_path, chunked=False, obj = self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, extra=extra, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) self.assertEqual(obj.name, 'foo_test_upload') self.assertEqual(obj.size, 1000) self.assertTrue('some-value' in obj.meta_data) @@ -564,8 +556,10 @@ def _foo_bar_container_foo_bar_object(self, method, url, body, headers): def _foo_bar_container_foo_test_upload_INVALID_HASH1(self, method, url, body, headers): body = '' + headers = {} + headers['etag'] = '"foobar"' # test_upload_object_invalid_hash1 - return (httplib.BAD_REQUEST, + return (httplib.OK, body, headers, httplib.responses[httplib.OK]) diff --git a/test/storage/test_swift.py b/test/storage/test_swift.py new file mode 100644 index 0000000000..e08e60ef1c --- /dev/null +++ b/test/storage/test_swift.py @@ -0,0 +1,658 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import os.path # pylint: disable-msg=W0404 +import sys +import copy +import unittest +import httplib + +import libcloud.utils + +from libcloud.common.types import LibcloudError, MalformedResponseError +from libcloud.storage.base import Container, Object +from libcloud.storage.types import ContainerAlreadyExistsError +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ContainerIsNotEmptyError +from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError +from libcloud.storage.types import InvalidContainerNameError +from libcloud.storage.drivers.swift import SwiftStorageDriver +from libcloud.storage.drivers.dummy import DummyIterator + +from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 +from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 + +current_hash = None + +class SwiftTests(unittest.TestCase): + + def setUp(self): + SwiftStorageDriver.connectionCls.conn_classes = ( + None, SwiftMockHttp) + SwiftStorageDriver.connectionCls.rawResponseCls = \ + SwiftMockRawResponse + SwiftMockHttp.type = None + SwiftMockRawResponse.type = None + self.driver = SwiftStorageDriver('dummy', 'dummy') + self._remove_test_file() + + def tearDown(self): + self._remove_test_file() + + def test_invalid_json_throws_exception(self): + SwiftMockHttp.type = 'MALFORMED_JSON' + try: + self.driver.list_containers() + except MalformedResponseError: + pass + else: + self.fail('Exception was not thrown') + + def test_list_containers(self): + SwiftMockHttp.type = 'EMPTY' + containers = self.driver.list_containers() + self.assertEqual(len(containers), 0) + + SwiftMockHttp.type = None + containers = self.driver.list_containers() + self.assertEqual(len(containers), 3) + + container = [c for c in containers if c.name == 'container2'][0] + self.assertEqual(container.extra['object_count'], 120) + self.assertEqual(container.extra['size'], 340084450) + + def test_list_container_objects(self): + SwiftMockHttp.type = 'EMPTY' + container = Container( + name='test_container', extra={}, driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 0) + + SwiftMockHttp.type = None + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 4) + + obj = [o for o in objects if o.name == 'foo test 1'][0] + self.assertEqual(obj.hash, '16265549b5bda64ecdaa5156de4c97cc') + self.assertEqual(obj.size, 1160520) + self.assertEqual(obj.container.name, 'test_container') + + def test_get_container(self): + container = self.driver.get_container(container_name='test_container') + self.assertEqual(container.name, 'test_container') + self.assertEqual(container.extra['object_count'], 800) + self.assertEqual(container.extra['size'], 1234568) + + def test_get_container_not_found(self): + try: + self.driver.get_container(container_name='not_found') + except ContainerDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_get_object_success(self): + obj = self.driver.get_object(container_name='test_container', + object_name='test_object') + self.assertEqual(obj.container.name, 'test_container') + self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') + self.assertEqual(obj.extra['content_type'], 'application/zip') + self.assertEqual( + obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') + self.assertEqual(obj.meta_data['foo-bar'], 'test 1') + self.assertEqual(obj.meta_data['bar-foo'], 'test 2') + + def test_get_object_not_found(self): + try: + self.driver.get_object(container_name='test_container', + object_name='not_found') + except ObjectDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_create_container_success(self): + container = self.driver.create_container( + container_name='test_create_container') + self.assertTrue(isinstance(container, Container)) + self.assertEqual(container.name, 'test_create_container') + self.assertEqual(container.extra['object_count'], 0) + + def test_create_container_already_exists(self): + SwiftMockHttp.type = 'ALREADY_EXISTS' + + try: + self.driver.create_container( + container_name='test_create_container') + except ContainerAlreadyExistsError: + pass + else: + self.fail( + 'Container already exists but an exception was not thrown') + + def test_create_container_invalid_name_too_long(self): + name = ''.join([ 'x' for x in range(0, 257)]) + try: + self.driver.create_container(container_name=name) + except InvalidContainerNameError: + pass + else: + self.fail( + 'Invalid name was provided (name is too long)' + ', but exception was not thrown') + + def test_create_container_invalid_name_slashes_in_name(self): + try: + self.driver.create_container(container_name='test/slashes/') + except InvalidContainerNameError: + pass + else: + self.fail( + 'Invalid name was provided (name contains slashes)' + ', but exception was not thrown') + + def test_delete_container_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + result = self.driver.delete_container(container=container) + self.assertTrue(result) + + def test_delete_container_not_found(self): + SwiftMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerDoesNotExistError: + pass + else: + self.fail( + 'Container does not exist but an exception was not thrown') + + def test_delete_container_not_empty(self): + SwiftMockHttp.type = 'NOT_EMPTY' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerIsNotEmptyError: + pass + else: + self.fail('Container is not empty but an exception was not thrown') + + def test_download_object_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertTrue(result) + + def test_download_object_invalid_file_size(self): + SwiftMockRawResponse.type = 'INVALID_SIZE' + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertFalse(result) + + def test_download_object_success_not_found(self): + SwiftMockRawResponse.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, + meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + try: + self.driver.download_object( + obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_download_object_as_stream(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + + stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) + self.assertTrue(hasattr(stream, '__iter__')) + + def test_upload_object_success(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + old_func = SwiftStorageDriver._upload_file + SwiftStorageDriver._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + extra = {'meta_data': { 'some-value': 'foobar'}} + obj = self.driver.upload_object(file_path=file_path, container=container, + extra=extra, object_name=object_name) + self.assertEqual(obj.name, 'foo_test_upload') + self.assertEqual(obj.size, 1000) + self.assertTrue('some-value' in obj.meta_data) + SwiftStorageDriver._upload_file = old_func + + def test_upload_object_invalid_hash(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + SwiftMockRawResponse.type = 'INVALID_HASH' + + old_func = SwiftStorageDriver._upload_file + SwiftStorageDriver._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object(file_path=file_path, container=container, + object_name=object_name, + verify_hash=True) + except ObjectHashMismatchError: + pass + else: + self.fail( + 'Invalid hash was returned but an exception was not thrown') + finally: + SwiftStorageDriver._upload_file = old_func + + def test_upload_object_no_content_type(self): + def no_content_type(name): + return None, None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = no_content_type + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object(file_path=file_path, container=container, + object_name=object_name) + except AttributeError: + pass + else: + self.fail( + 'File content type not provided' + ' but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_error(self): + def dummy_content_type(name): + return 'application/zip', None + + def send(instance): + raise Exception('') + + old_func1 = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + old_func2 = SwiftMockHttp.send + SwiftMockHttp.send = send + + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except LibcloudError: + pass + else: + self.fail('Timeout while uploading but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func1 + SwiftMockHttp.send = old_func2 + + def test_upload_object_inexistent_file(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + file_path = os.path.abspath(__file__ + '.inexistent') + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except OSError: + pass + else: + self.fail('Inesitent but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_via_stream(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_stream_data' + iterator = DummyIterator(data=['2', '3', '5']) + try: + self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator) + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_delete_object_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + status = self.driver.delete_object(obj=obj) + self.assertTrue(status) + + def test_delete_object_not_found(self): + SwiftMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + try: + self.driver.delete_object(obj=obj) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_ex_get_meta_data(self): + meta_data = self.driver.ex_get_meta_data() + self.assertTrue(isinstance(meta_data, dict)) + self.assertTrue('object_count' in meta_data) + self.assertTrue('container_count' in meta_data) + self.assertTrue('bytes_used' in meta_data) + + def _remove_test_file(self): + file_path = os.path.abspath(__file__) + '.temp' + + try: + os.unlink(file_path) + except OSError: + pass + +class SwiftMockHttp(StorageMockHttp): + + fixtures = StorageFileFixtures('swift') + base_headers = { 'content-type': 'application/json; charset=UTF-8'} + + # fake auth token response + def _v1_0(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + headers.update({ 'x-auth-token': 'FE011C19', + 'x-storage-token': 'FE011C19', + 'x-storage-url': + 'https://storage4.clouddrive.com/v1/MossoCloudFS'}) + return (httplib.NO_CONTENT, + "", + headers, + httplib.responses[httplib.NO_CONTENT]) + + def _v1_MossoCloudFS_MALFORMED_JSON(self, method, url, body, headers): + # test_invalid_json_throws_exception + body = 'broken: json /*"' + return (httplib.NO_CONTENT, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_EMPTY(self, method, url, body, headers): + return (httplib.NO_CONTENT, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'GET': + # list_containers + body = self.fixtures.load('list_containers.json') + status_code = httplib.OK + elif method == 'HEAD': + # get_meta_data + body = self.fixtures.load('meta_data.json') + status_code = httplib.NO_CONTENT + headers.update({ 'x-account-container-count': 10, + 'x-account-object-count': 400, + 'x-account-bytes-used': 1234567 + }) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_not_found(self, method, url, body, headers): + # test_get_object_not_found + if method == 'HEAD': + body = '' + else: + raise ValueError('Invalid method') + + return (httplib.NOT_FOUND, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_EMPTY(self, method, url, body, headers): + body = self.fixtures.load('list_container_objects_empty.json') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'GET': + # list_container_objects + body = self.fixtures.load('list_container_objects.json') + status_code = httplib.OK + elif method == 'HEAD': + # get_container + body = self.fixtures.load('list_container_objects_empty.json') + status_code = httplib.NO_CONTENT + headers.update({ 'x-container-object-count': 800, + 'x-container-bytes-used': 1234568 + }) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_not_found( + self, method, url, body, headers): + # test_get_container_not_found + if method == 'HEAD': + body = '' + else: + raise ValueError('Invalid method') + + return (httplib.NOT_FOUND, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_test_object( + self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'HEAD': + # get_object + body = self.fixtures.load('list_container_objects_empty.json') + status_code = httplib.NO_CONTENT + headers.update({ 'content-length': 555, + 'last-modified': 'Tue, 25 Jan 2011 22:01:49 GMT', + 'etag': '6b21c4a111ac178feacf9ec9d0c71f17', + 'x-object-meta-foo-bar': 'test 1', + 'x-object-meta-bar-foo': 'test 2', + 'content-type': 'application/zip'}) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_create_container( + self, method, url, body, headers): + # test_create_container_success + headers = copy.deepcopy(self.base_headers) + body = self.fixtures.load('list_container_objects_empty.json') + headers = copy.deepcopy(self.base_headers) + headers.update({ 'content-length': 18, + 'date': 'Mon, 28 Feb 2011 07:52:57 GMT' + }) + status_code = httplib.CREATED + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_create_container_ALREADY_EXISTS( + self, method, url, body, headers): + # test_create_container_already_exists + headers = copy.deepcopy(self.base_headers) + body = self.fixtures.load('list_container_objects_empty.json') + headers.update({ 'content-type': 'text/plain' }) + status_code = httplib.ACCEPTED + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container(self, method, url, body, headers): + if method == 'DELETE': + # test_delete_container_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_NOT_FOUND( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_container_not_found + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NOT_FOUND + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_NOT_EMPTY( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_container_not_empty + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.CONFLICT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_object_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_object_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NOT_FOUND + + return (status_code, body, headers, httplib.responses[httplib.OK]) + +class SwiftMockRawResponse(MockRawResponse): + + fixtures = StorageFileFixtures('swift') + base_headers = { 'content-type': 'application/json; charset=UTF-8'} + + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( + self, method, url, body, headers): + # test_object_upload_success + + body = '' + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'hash343hhash89h932439jsaa89' + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( + self, method, url, body, headers): + # test_object_upload_invalid_hash + body = '' + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'foobar' + return (httplib.CREATED, body, headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( + self, method, url, body, headers): + + # test_download_object_success + body = 'test' + self._data = self._generate_random_data(1000) + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_INVALID_SIZE( + self, method, url, body, headers): + # test_download_object_invalid_file_size + body = 'test' + self._data = self._generate_random_data(100) + return (httplib.OK, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( + self, method, url, body, headers): + body = '' + return (httplib.NOT_FOUND, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data( + self, method, url, body, headers): + + # test_upload_object_via_stream_success + headers = {} + headers.update(self.base_headers) + headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e' + body = 'test' + return (httplib.CREATED, + body, + headers, + httplib.responses[httplib.OK]) + +if __name__ == '__main__': + sys.exit(unittest.main())