import boto3
session = boto3.Session(profile_name='slscode')
s3 = session.client('s3')
s3.list_buckets()
{'Buckets': [{'CreationDate': datetime.datetime(2016, 11, 17, 1, 52, 54, tzinfo=tzutc()),
'Name': 'base-logbucket-1p2yu1w4mcn6s'},
{'CreationDate': datetime.datetime(2016, 11, 17, 1, 52, 54, tzinfo=tzutc()),
'Name': 'base-storagebucket-v6ohdrq7cgkn'},
{'CreationDate': datetime.datetime(2017, 6, 21, 22, 35, 4, tzinfo=tzutc()),
'Name': 'catfacts-dev-serverlessdeploymentbucket-1oj1khetpsony'},
{'CreationDate': datetime.datetime(2016, 10, 20, 23, 36, 34, tzinfo=tzutc()),
'Name': 'cf-templates-1jhk9i63f7204-us-east-1'},
{'CreationDate': datetime.datetime(2016, 8, 24, 13, 54, 51, tzinfo=tzutc()),
'Name': 'cf-templates-1jhk9i63f7204-us-west-2'},
{'CreationDate': datetime.datetime(2017, 2, 5, 19, 20, 56, tzinfo=tzutc()),
'Name': 'crop-catalog-assets'},
{'CreationDate': datetime.datetime(2017, 4, 8, 15, 44, 53, tzinfo=tzutc()),
'Name': 'gladnews-dev-serverlessdeploymentbucket-124ei9eqddlzk'},
{'CreationDate': datetime.datetime(2017, 4, 8, 15, 45, 34, tzinfo=tzutc()),
'Name': 'gladnews-dev-website-hwbxhst99j7i'},
{'CreationDate': datetime.datetime(2017, 2, 25, 20, 43, 14, tzinfo=tzutc()),
'Name': 'logs-perf-test'},
{'CreationDate': datetime.datetime(2016, 8, 25, 18, 17, 33, tzinfo=tzutc()),
'Name': 'ryansb-ansible-scratch'},
{'CreationDate': datetime.datetime(2016, 8, 24, 12, 53, 38, tzinfo=tzutc()),
'Name': 'sls-templates'},
{'CreationDate': datetime.datetime(2017, 4, 12, 23, 32, 7, tzinfo=tzutc()),
'Name': 'stage-dev-serverlessdeploymentbucket-48nurxoae887'},
{'CreationDate': datetime.datetime(2017, 4, 12, 23, 32, 44, tzinfo=tzutc()),
'Name': 'stage-dev-website-15r4h9rmmvbti'},
{'CreationDate': datetime.datetime(2017, 3, 24, 23, 48, 40, tzinfo=tzutc()),
'Name': 'urlshortener-prod-serverlessdeploymentbucket-qc2imnrfhppj'}],
'Owner': {'DisplayName': 'sb',
'ID': '6e1c1b27e7b2260601a221de952e884528fd00a83d08ee0370eb08748d150934'},
'ResponseMetadata': {'HTTPHeaders': {'content-type': 'application/xml',
'date': 'Fri, 23 Jun 2017 12:50:54 GMT',
'server': 'AmazonS3',
'transfer-encoding': 'chunked',
'x-amz-id-2': 'I8LR/7gninyS3kADE7sM20Aui9coSQDwDtbnrygFTfygrV4Jy7CMT1xEN/YiCDWBqQIfmhHyqtM=',
'x-amz-request-id': 'CA473A446DAEFAC8'},
'HTTPStatusCode': 200,
'HostId': 'I8LR/7gninyS3kADE7sM20Aui9coSQDwDtbnrygFTfygrV4Jy7CMT1xEN/YiCDWBqQIfmhHyqtM=',
'RequestId': 'CA473A446DAEFAC8',
'RetryAttempts': 0}}
s3.list_objects_v2(Bucket='crop-catalog-assets')
{'Contents': [{'ETag': '"682e6ee3bb193650b06616e855376ee8"',
'Key': '/templates/template-2.json',
'LastModified': datetime.datetime(2017, 2, 17, 2, 2, tzinfo=tzutc()),
'Size': 3123,
'StorageClass': 'STANDARD'},
{'ETag': '"742d140d582c161078de00ec434e3c06"',
'Key': '/zipfiles/crop-sample.zip',
'LastModified': datetime.datetime(2017, 2, 17, 2, 2, tzinfo=tzutc()),
'Size': 849,
'StorageClass': 'STANDARD'},
{'ETag': '"742d140d582c161078de00ec434e3c06"',
'Key': 'assets/prod-7qedu6ehkucdi/crop-sample.zip',
'LastModified': datetime.datetime(2017, 2, 17, 1, 59, 6, tzinfo=tzutc()),
'Size': 849,
'StorageClass': 'STANDARD'},
{'ETag': '"9893fb04c0ac07cea648605705abc507"',
'Key': 'crop-project/templates/template-2.json',
'LastModified': datetime.datetime(2017, 2, 17, 2, 3, 3, tzinfo=tzutc()),
'Size': 3135,
'StorageClass': 'STANDARD'},
{'ETag': '"742d140d582c161078de00ec434e3c06"',
'Key': 'crop-project/zipfiles/crop-sample.zip',
'LastModified': datetime.datetime(2017, 2, 17, 2, 3, 3, tzinfo=tzutc()),
'Size': 849,
'StorageClass': 'STANDARD'},
{'ETag': '"2498aafa24f784e7f6a3fa0a4c9084f6"',
'Key': 'prod-444eg4gz2kjyg/assets//template.json',
'LastModified': datetime.datetime(2017, 2, 6, 0, 17, 4, tzinfo=tzutc()),
'Size': 3443,
'StorageClass': 'STANDARD'},
{'ETag': '"2c11d66f42628e4b304cd5459548a14d"',
'Key': 'prod-444eg4gz2kjyg/assets/crop-sample.zip',
'LastModified': datetime.datetime(2017, 2, 7, 0, 29, 4, tzinfo=tzutc()),
'Size': 639,
'StorageClass': 'STANDARD'},
{'ETag': '"7a6d8772c4cfccebe1a52bd037733a81"',
'Key': 'prod-444eg4gz2kjyg/assets/template.json',
'LastModified': datetime.datetime(2017, 2, 7, 0, 29, 5, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"2c11d66f42628e4b304cd5459548a14d"',
'Key': 'prod-7qedu6ehkucdi/assets/crop-sample.zip',
'LastModified': datetime.datetime(2017, 2, 7, 19, 5, 46, tzinfo=tzutc()),
'Size': 639,
'StorageClass': 'STANDARD'},
{'ETag': '"5381f231326641b05b3dd64f260df9c7"',
'Key': 'prod-7qedu6ehkucdi/assets/template-3.json',
'LastModified': datetime.datetime(2017, 2, 7, 19, 5, 47, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"e59092299c872f13de8ad15c6e91044e"',
'Key': 'prod-7qedu6ehkucdi/assets/template.json',
'LastModified': datetime.datetime(2017, 2, 7, 1, 9, 32, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"f3ff047ad5000dcf1e4930c677496c05"',
'Key': 'templates/prod-7qedu6ehkucdi/template-2.json',
'LastModified': datetime.datetime(2017, 2, 17, 1, 59, 6, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'Key': 'templates/prod-7qedu6ehkucdi/template-22.json',
'LastModified': datetime.datetime(2017, 2, 17, 1, 47, 10, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"1e92362cdbefcbeaef6edf93528dee36"',
'Key': 'templates/prod-7qedu6ehkucdi/template-3.json',
'LastModified': datetime.datetime(2017, 2, 7, 19, 13, 4, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'},
{'ETag': '"4356b4f198547d65e546bcd861629558"',
'Key': 'templates/prod-7qedu6ehkucdi/template-4.json',
'LastModified': datetime.datetime(2017, 2, 7, 19, 26, 31, tzinfo=tzutc()),
'Size': 3139,
'StorageClass': 'STANDARD'}],
'IsTruncated': False,
'KeyCount': 15,
'MaxKeys': 1000,
'Name': 'crop-catalog-assets',
'Prefix': '',
'ResponseMetadata': {'HTTPHeaders': {'content-type': 'application/xml',
'date': 'Fri, 23 Jun 2017 12:50:55 GMT',
'server': 'AmazonS3',
'transfer-encoding': 'chunked',
'x-amz-bucket-region': 'us-east-1',
'x-amz-id-2': 'wq2HjUfpyeTbzmmU5g/ecx/BtO7fIEUU/FUGl7HUjxRS9hcwqKlVwq5bhx0vgJt1o3QDmzPOGZQ=',
'x-amz-request-id': '86A3E6A0A855103E'},
'HTTPStatusCode': 200,
'HostId': 'wq2HjUfpyeTbzmmU5g/ecx/BtO7fIEUU/FUGl7HUjxRS9hcwqKlVwq5bhx0vgJt1o3QDmzPOGZQ=',
'RequestId': '86A3E6A0A855103E',
'RetryAttempts': 1}}
template_body = s3.get_object(Bucket='crop-catalog-assets', Key='templates/prod-7qedu6ehkucdi/template-22.json')
template_body
{'AcceptRanges': 'bytes',
'Body': <botocore.response.StreamingBody at 0x7f717b9ebf28>,
'ContentLength': 3139,
'ContentType': 'application/json',
'ETag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'Expiration': 'expiry-date="Wed, 04 Apr 2018 00:00:00 GMT", rule-id="remove-old-versions"',
'LastModified': datetime.datetime(2017, 2, 17, 1, 47, 10, tzinfo=tzutc()),
'Metadata': {},
'ResponseMetadata': {'HTTPHeaders': {'accept-ranges': 'bytes',
'content-length': '3139',
'content-type': 'application/json',
'date': 'Fri, 23 Jun 2017 12:50:55 GMT',
'etag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'last-modified': 'Fri, 17 Feb 2017 01:47:10 GMT',
'server': 'AmazonS3',
'x-amz-expiration': 'expiry-date="Wed, 04 Apr 2018 00:00:00 GMT", rule-id="remove-old-versions"',
'x-amz-id-2': 'ZqgQZpi6BuRAWY2gK7dtdwSSvgyqEPfl4Svy6zT1+FT0HUtApWMNEKneJZefI+wPAZ0XP6GQXog=',
'x-amz-request-id': '6CA06FF772184D6B',
'x-amz-version-id': 'LteUEWrHlL39kQZqjbVjGap28BCoyIBM'},
'HTTPStatusCode': 200,
'HostId': 'ZqgQZpi6BuRAWY2gK7dtdwSSvgyqEPfl4Svy6zT1+FT0HUtApWMNEKneJZefI+wPAZ0XP6GQXog=',
'RequestId': '6CA06FF772184D6B',
'RetryAttempts': 0},
'VersionId': 'LteUEWrHlL39kQZqjbVjGap28BCoyIBM'}
import json
parsed_template = json.load(template_body['Body'])
print(type(parsed_template))
parsed_template
<class 'dict'>
{'AWSTemplateFormatVersion': '2010-09-09',
'Description': 'The AWS CloudFormation template for this Serverless application',
'Outputs': {'ServiceEndpoint': {'Description': 'URL of the service endpoint',
'Value': {'Fn::Join': ['',
['https://',
{'Ref': 'ApiGatewayRestApi'},
'.execute-api.us-east-1.amazonaws.com/dev']]}}},
'Resources': {'ApiGatewayDeployment1487296027700': {'DependsOn': ['ApiGatewayMethodHiGet'],
'Properties': {'RestApiId': {'Ref': 'ApiGatewayRestApi'},
'StageName': 'dev'},
'Type': 'AWS::ApiGateway::Deployment'},
'ApiGatewayMethodHiGet': {'Properties': {'AuthorizationType': 'NONE',
'HttpMethod': 'GET',
'Integration': {'IntegrationHttpMethod': 'POST',
'Type': 'AWS_PROXY',
'Uri': {'Fn::Join': ['',
['arn:aws:apigateway:',
{'Ref': 'AWS::Region'},
':lambda:path/2015-03-31/functions/',
{'Fn::GetAtt': ['HelloLambdaFunction', 'Arn']},
'/invocations']]}},
'MethodResponses': [],
'RequestParameters': {},
'ResourceId': {'Ref': 'ApiGatewayResourceHi'},
'RestApiId': {'Ref': 'ApiGatewayRestApi'}},
'Type': 'AWS::ApiGateway::Method'},
'ApiGatewayResourceHi': {'Properties': {'ParentId': {'Fn::GetAtt': ['ApiGatewayRestApi',
'RootResourceId']},
'PathPart': 'hi',
'RestApiId': {'Ref': 'ApiGatewayRestApi'}},
'Type': 'AWS::ApiGateway::Resource'},
'ApiGatewayRestApi': {'Properties': {'Name': 'dev-crop-sample'},
'Type': 'AWS::ApiGateway::RestApi'},
'HelloLambdaFunction': {'DependsOn': ['IamPolicyLambdaExecution',
'IamRoleLambdaExecution'],
'Properties': {'Code': {'S3Bucket': 'crop-catalog-assets',
'S3Key': 'assets/prod-7qedu6ehkucdi/crop-sample.zip',
'S3ObjectVersion': '4ZFOBBIpBGxcN4xVFkj2I3lrGjdwh1F8'},
'FunctionName': 'crop-sample-dev-hello',
'Handler': 'handler.hello',
'MemorySize': 1024,
'Role': {'Fn::GetAtt': ['IamRoleLambdaExecution', 'Arn']},
'Runtime': 'python2.7',
'Timeout': 6},
'Type': 'AWS::Lambda::Function'},
'HelloLambdaPermissionApiGateway': {'Properties': {'Action': 'lambda:InvokeFunction',
'FunctionName': {'Fn::GetAtt': ['HelloLambdaFunction', 'Arn']},
'Principal': 'apigateway.amazonaws.com',
'SourceArn': {'Fn::Join': ['',
['arn:aws:execute-api:',
{'Ref': 'AWS::Region'},
':',
{'Ref': 'AWS::AccountId'},
':',
{'Ref': 'ApiGatewayRestApi'},
'/*/*']]}},
'Type': 'AWS::Lambda::Permission'},
'HelloLogGroup': {'Properties': {'LogGroupName': '/aws/lambda/crop-sample-dev-hello'},
'Type': 'AWS::Logs::LogGroup'},
'IamPolicyLambdaExecution': {'DependsOn': ['IamRoleLambdaExecution'],
'Properties': {'PolicyDocument': {'Statement': [{'Action': ['logs:CreateLogStream',
'logs:PutLogEvents'],
'Effect': 'Allow',
'Resource': [{'Fn::Join': [':',
[{'Fn::GetAtt': ['HelloLogGroup', 'Arn']}, '*']]}]}],
'Version': '2012-10-17'},
'PolicyName': {'Fn::Join': ['-', ['dev', 'crop-sample', 'lambda']]},
'Roles': [{'Ref': 'IamRoleLambdaExecution'}]},
'Type': 'AWS::IAM::Policy'},
'IamRoleLambdaExecution': {'Properties': {'AssumeRolePolicyDocument': {'Statement': [{'Action': ['sts:AssumeRole'],
'Effect': 'Allow',
'Principal': {'Service': ['lambda.amazonaws.com']}}],
'Version': '2012-10-17'},
'Path': '/',
'RoleName': {'Fn::Join': ['-',
['crop-sample', 'dev', 'us-east-1', 'lambdaRole']]}},
'Type': 'AWS::IAM::Role'}}}
resp = s3.put_object(
Bucket='ryansb-ansible-scratch',
Key='test.json',
ACL='public-read',
ContentType='application/json',
StorageClass='REDUCED_REDUNDANCY',
Body=json.dumps(parsed_template)
)
resp
{'ETag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'Expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'ResponseMetadata': {'HTTPHeaders': {'content-length': '0',
'date': 'Fri, 23 Jun 2017 12:50:55 GMT',
'etag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'server': 'AmazonS3',
'x-amz-expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'x-amz-id-2': 'O1GhtZxg0EZ6Xi9vFhMHRgqVzzEEC/1SErOq/cICRKhLHTTZhj9ur+laUXFcgA9uyUEFdnpz/oY=',
'x-amz-request-id': 'DCCE0235AACA1753',
'x-amz-storage-class': 'REDUCED_REDUNDANCY'},
'HTTPStatusCode': 200,
'HostId': 'O1GhtZxg0EZ6Xi9vFhMHRgqVzzEEC/1SErOq/cICRKhLHTTZhj9ur+laUXFcgA9uyUEFdnpz/oY=',
'RequestId': 'DCCE0235AACA1753',
'RetryAttempts': 0}}
import gzip
with gzip.open('/tmp/tpl.json.gz', mode='wb', compresslevel=9) as compress:
s3.download_fileobj(
Bucket='crop-catalog-assets',
Key='templates/prod-7qedu6ehkucdi/template-22.json',
Fileobj=compress)
resp = s3.put_object(
Bucket='ryansb-ansible-scratch',
Key='test.json.gz',
ACL='public-read',
ContentType='application/json',
ContentEncoding='gzip',
StorageClass='REDUCED_REDUNDANCY',
Body=open('/tmp/tpl.json.gz', 'rb')
)
resp
{'ETag': '"3fdc8902a90aa81578bc6d5d00c22c9d"',
'Expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'ResponseMetadata': {'HTTPHeaders': {'content-length': '0',
'date': 'Fri, 23 Jun 2017 12:50:56 GMT',
'etag': '"3fdc8902a90aa81578bc6d5d00c22c9d"',
'server': 'AmazonS3',
'x-amz-expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'x-amz-id-2': 'yhK1/OyTPZ0TTD8rgdIEl9E4/gmzvy8gGBRcC7GofjWLaWHWwTkJzQdxRTte/YpLD0JH/u5zurM=',
'x-amz-request-id': 'C55AB0831CFECE92',
'x-amz-storage-class': 'REDUCED_REDUNDANCY'},
'HTTPStatusCode': 200,
'HostId': 'yhK1/OyTPZ0TTD8rgdIEl9E4/gmzvy8gGBRcC7GofjWLaWHWwTkJzQdxRTte/YpLD0JH/u5zurM=',
'RequestId': 'C55AB0831CFECE92',
'RetryAttempts': 0}}
with open('/tmp/tpl.json.gz', 'rb') as fh: print(len(fh.read()))
1122
s3.head_object(Bucket='ryansb-ansible-scratch', Key='test.json')
{'AcceptRanges': 'bytes',
'ContentLength': 3139,
'ContentType': 'application/json',
'ETag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'Expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'LastModified': datetime.datetime(2017, 6, 23, 12, 50, 55, tzinfo=tzutc()),
'Metadata': {},
'ResponseMetadata': {'HTTPHeaders': {'accept-ranges': 'bytes',
'content-length': '3139',
'content-type': 'application/json',
'date': 'Fri, 23 Jun 2017 12:50:56 GMT',
'etag': '"f7bdb431b9b3719f4bce36d9c1eb58a1"',
'last-modified': 'Fri, 23 Jun 2017 12:50:55 GMT',
'server': 'AmazonS3',
'x-amz-expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'x-amz-id-2': 'Xdyng9xkVBP8MqMNFmOSFzVVneRTrb+3/qNwGKBZws+y1VMhd2eHRMnPf2GuDpKMM/NNAPbnhwI=',
'x-amz-request-id': '777F2C28C378D0C8',
'x-amz-storage-class': 'REDUCED_REDUNDANCY'},
'HTTPStatusCode': 200,
'HostId': 'Xdyng9xkVBP8MqMNFmOSFzVVneRTrb+3/qNwGKBZws+y1VMhd2eHRMnPf2GuDpKMM/NNAPbnhwI=',
'RequestId': '777F2C28C378D0C8',
'RetryAttempts': 0},
'StorageClass': 'REDUCED_REDUNDANCY'}
s3.head_object(Bucket='ryansb-ansible-scratch', Key='test.json.gz')
{'AcceptRanges': 'bytes',
'ContentEncoding': 'gzip',
'ContentLength': 1122,
'ContentType': 'application/json',
'ETag': '"3fdc8902a90aa81578bc6d5d00c22c9d"',
'Expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'LastModified': datetime.datetime(2017, 6, 23, 12, 50, 56, tzinfo=tzutc()),
'Metadata': {},
'ResponseMetadata': {'HTTPHeaders': {'accept-ranges': 'bytes',
'content-encoding': 'gzip',
'content-length': '1122',
'content-type': 'application/json',
'date': 'Fri, 23 Jun 2017 12:50:56 GMT',
'etag': '"3fdc8902a90aa81578bc6d5d00c22c9d"',
'last-modified': 'Fri, 23 Jun 2017 12:50:56 GMT',
'server': 'AmazonS3',
'x-amz-expiration': 'expiry-date="Sat, 08 Jul 2017 00:00:00 GMT", rule-id="NukeFromOrbit"',
'x-amz-id-2': 'u2HFIU8eXGncj8NUEpnmolFg1nHekYvXULjxcJs4lHbuHCiO01bpMhKZ4VEjfy48ztVudr8n/gM=',
'x-amz-request-id': '24F83E566E242E76',
'x-amz-storage-class': 'REDUCED_REDUNDANCY'},
'HTTPStatusCode': 200,
'HostId': 'u2HFIU8eXGncj8NUEpnmolFg1nHekYvXULjxcJs4lHbuHCiO01bpMhKZ4VEjfy48ztVudr8n/gM=',
'RequestId': '24F83E566E242E76',
'RetryAttempts': 0},
'StorageClass': 'REDUCED_REDUNDANCY'}
bukkit = session.resource('s3').Bucket('ryansb-ansible-scratch')
for obj in bukkit.objects.all():
print(obj.key, obj.e_tag, obj.size)
if obj.size > 2**30: obj.delete()
test.json "f7bdb431b9b3719f4bce36d9c1eb58a1" 3139 test.json.gz "3fdc8902a90aa81578bc6d5d00c22c9d" 1122