#!/usr/bin/python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Version @VERSION@ # # A plugin for executing script needed by cloud stack from __future__ import with_statement from copy import copy from datetime import datetime from httplib import * from string import join import os import sys import time import hashlib import base64 import hmac import traceback import urllib2 import XenAPIPlugin sys.path.extend(["/opt/xensource/sm/"]) import util NULL = 'null' # Value conversion utility functions ... def to_none(value): return value if value is not None and value.strip() != NULL else None def to_bool(value): return True if to_none(value) in ['true', 'True', None] else False def to_integer(value, default): return int(value) if to_none(value) is not None else default def optional_str_value(value, default): return value if is_not_blank(value) else default def is_not_blank(value): return True if to_none(value) is not None and value.strip != '' else False def get_optional_key(map, key, default=''): return map[key] if key in map else default def log(message): util.SMlog('#### VMOPS %s ####' % message) def echo(fn): def wrapped(*v, **k): name = fn.__name__ log("enter %s ####" % name) res = fn(*v, **k) log("exit %s with result %s" % name, res) return res return wrapped def require_str_value(value, error_message): if is_not_blank(value): return value raise ValueError(error_message) def retry(max_attempts, fn): attempts = 1 while attempts <= max_attempts: log("Attempting execution {0}/{1} of {2}". format(attempts, max_attempts, fn.__name__)) try: return fn() except: if (attempts >= max_attempts): raise attempts = attempts + 1 def compute_md5(filename, buffer_size=8192): hasher = hashlib.md5() with open(filename, 'rb') as file: data = file.read(buffer_size) while data != "": hasher.update(data) data = file.read(buffer_size) return base64.encodestring(hasher.digest())[:-1] class S3Client(object): DEFAULT_END_POINT = 's3.amazonaws.com' DEFAULT_CONNECTION_TIMEOUT = 50000 DEFAULT_SOCKET_TIMEOUT = 50000 DEFAULT_MAX_ERROR_RETRY = 3 HEADER_CONTENT_MD5 = 'Content-MD5' HEADER_CONTENT_TYPE = 'Content-Type' HEADER_CONTENT_LENGTH = 'Content-Length' def __init__(self, access_key, secret_key, end_point=None, https_flag=None, connection_timeout=None, socket_timeout=None, max_error_retry=None): self.access_key = require_str_value( access_key, 'An access key must be specified.') self.secret_key = require_str_value( secret_key, 'A secret key must be specified.') self.end_point = optional_str_value(end_point, self.DEFAULT_END_POINT) self.https_flag = to_bool(https_flag) self.connection_timeout = to_integer( connection_timeout, self.DEFAULT_CONNECTION_TIMEOUT) self.socket_timeout = to_integer( socket_timeout, self.DEFAULT_SOCKET_TIMEOUT) self.max_error_retry = to_integer( max_error_retry, self.DEFAULT_MAX_ERROR_RETRY) def build_canocialized_resource(self, bucket, key): return '/{bucket}/{key}'.format(bucket=bucket, key=key) def noop_send_body(): pass def noop_read(response): return response.read() def do_operation( self, method, bucket, key, input_headers={}, fn_send_body=noop_send_body, fn_read=noop_read): headers = copy(input_headers) headers['Expect'] = '100-continue' uri = self.build_canocialized_resource(bucket, key) signature, request_date = self.sign_request(method, uri, headers) headers['Authorization'] = "AWS {0}:{1}".format( self.access_key, signature) headers['Date'] = request_date connection = HTTPSConnection(self.end_point) \ if self.https_flag else HTTPConnection(self.end_point) connection.timeout = self.socket_timeout def perform_request(): connection.request(method, uri, fn_send_body(), headers) response = connection.getresponse() log("Sent {0} request to {1} {2} with headers {3}. \ Got response status {4}: {5}". format(method, self.end_point, uri, headers, response.status, response.reason)) return fn_read(response) try: return retry(self.max_error_retry, perform_request) finally: connection.close() ''' See http://bit.ly/MMC5de for more information regarding the creation of AWS authorization tokens and header signing ''' def sign_request(self, operation, canocialized_resource, headers): request_date = datetime.utcnow( ).strftime('%a, %d %b %Y %H:%M:%S +0000') content_hash = get_optional_key(headers, self.HEADER_CONTENT_MD5) content_type = get_optional_key(headers, self.HEADER_CONTENT_TYPE) string_to_sign = join( [operation, content_hash, content_type, request_date, canocialized_resource], '\n') signature = base64.encodestring( hmac.new(self.secret_key, string_to_sign.encode('utf8'), hashlib.sha1).digest())[:-1] return signature, request_date def put(self, bucket, key, src_filename): headers = { self.HEADER_CONTENT_MD5: compute_md5(src_filename), self.HEADER_CONTENT_TYPE: 'application/octet-stream', self.HEADER_CONTENT_LENGTH: os.stat(src_filename).st_size, } def send_body(): return open(src_filename, 'rb') self.do_operation('PUT', bucket, key, headers, send_body) def get(self, bucket, key, target_filename): def read(response): with open(target_filename, 'wb') as file: while True: block = response.read(8192) if not block: break file.write(block) return self.do_operation('GET', bucket, key, fn_read=read) def delete(self, bucket, key): return self.do_operation('DELETE', bucket, key) def parseArguments(args): # The keys in the args map will correspond to the properties defined on # the com.cloud.utils.S3Utils#ClientOptions interface client = S3Client( args['accessKey'], args['secretKey'], args['endPoint'], args['isHttps'], args['connectionTimeout'], args['socketTimeout']) operation = args['operation'] bucket = args['bucket'] key = args['key'] filename = args['filename'] if is_blank(operation): raise ValueError('An operation must be specified.') if is_blank(bucket): raise ValueError('A bucket must be specified.') if is_blank(key): raise ValueError('A value must be specified.') if is_blank(filename): raise ValueError('A filename must be specified.') return client, operation, bucket, key, filename @echo def s3(session, args): client, operation, bucket, key, filename = parseArguments(args) try: if operation == 'put': client.put(bucket, key, filename) elif operation == 'get': client.get(bucket, key, filename) elif operation == 'delete': client.delete(bucket, key, filename) else: raise RuntimeError( "S3 plugin does not support operation {0}.".format(operation)) return 'true' except: log("Operation {0} on file {1} from/in bucket {2} key {3}.".format( operation, filename, bucket, key)) log(traceback.format_exc()) return 'false' if __name__ == "__main__": XenAPIPlugin.dispatch({"s3": s3})