Large Custom Images – 5GBを超えるファイルのFujitsu K5 Object Storage–(Swift)へのアップロード
2016-10-08
Machine-translated — the English original is authoritative.
以前のブログ投稿で、Fujitsu K5 IaaSプラットフォームにカスタムイメージをアップロードする簡単なプロセスについて詳しく説明しました。ここで私が見過ごしていた課題は、大きなカスタムイメージのサイズに関するもので、ここではそれについて対処します。
背景: OpenStack Swiftオブジェクトには5GBの最大サイズ制限があります。ただし、Swiftコンテナにはこれらのオブジェクトを数千個含めることができます。5GBを超えるファイルは、コンテナにアップロードする前に、より小さなファイルのサブセットに分割する必要があります。すべての構成ファイルがアップロードされると、'manifest'ヘッダーを持つ0バイトのファイルがコンテナにアップロードされます。このヘッダーは、構成ファイル名を構築するために使用されるコンテナ名とプレフィックスで構成されています。この0バイトのファイルがSwift APIを通じて参照されると、すべての構成ファイルが連結され、元の大きなファイルがダウンロードされます。
以下のPythonスクリプトは、1GB未満のカスタムイメージをK5のオブジェクトストレージに直接アップロードします。より大きなファイルは、オブジェクトストレージにアップロードする前に1GBのチャンクに分割されます。デフォルトの1GBのサイズは、コマンドラインパラメータを使用して変更できます。
イメージがアップロードされると、デフォルトのK5プロジェクトに登録されます。このイメージを消費したい他のメンバープロジェクトと共有する必要があります。これは次の投稿で共有されます。
前提条件 : スクリプトは設定ファイル k5contractsettings.py に依存しており、すべての契約詳細を含み、同じディレクトリに配置する必要があります。例えば:
このファイルには、以下に表示されるものとは異なって解釈またはコンパイルされる可能性がある、隠されたまたは双方向のUnicode文字が含まれています。確認するには、隠されたUnicode文字を表示するエディタでファイルを開きます。
双方向のUnicode文字について詳しく学ぶ
| #!/usr/bin/python | |
| adminUser = 'username' | |
| adminPassword = 'password' | |
| contract = 'contract_name' | |
| contractid = 'contract_id' | |
| defaultid = 'default_project_id' | |
| project = 'working_project' | |
| region = 'uk-1' |
rawを表示
k5contractsettings.py
❤️ GitHub によってホストされています
イメージアップロード例スクリプト
このファイルには、以下に表示されるものとは異なって解釈またはコンパイルされる可能性がある、隠されたまたは双方向のUnicode文字が含まれています。確認するには、隠されたUnicode文字を表示するエディタでファイルを開きます。
双方向のUnicode文字について詳しく学ぶ
| #!/usr/bin/python | |
| # Author : Graham Land | |
| # Date: 08/10/2016 | |
| # | |
| # Purpose: Upload a custom image to K5 Object Storage and then register with K5 Glance | |
| # If the image is greater than 1GB it will be broken down into 1GB chunks | |
| # and then uploaded | |
| # Command line parameters – | |
| # -i image_path | |
| # -c container_name | |
| # -s chunk_size (bytes) | |
| # -n display_name | |
| # -t image_type | |
| # -p project | |
| # | |
| # Prerequisites: k5contractsettings.py file in the same directory with login details | |
| # | |
| # adminUser = 'username' | |
| # adminPassword = 'password' | |
| # contract = 'contract_name' | |
| # contractid = 'contract_id' | |
| # defaultid = 'default_project_id' | |
| # project = 'working_project' | |
| # region = 'uk-1' | |
| # | |
| # blog: https://allthingscloud.eu | |
| # twitter: @allthingsclowd | |
| import sys | |
| import os | |
| import requests | |
| import uuid | |
| import base64 | |
| import time | |
| import getopt | |
| import ntpath | |
| # load your K5 contract details from k5contractsettings.py file | |
| from k5contractsettings import * | |
| # get a scoped auth token | |
| def get_scoped_token(uname,upassword,uproject,udomain,uregion): | |
| identityURL = 'https://identity.' + uregion + '.cloud.global.fujitsu.com/v3/auth/tokens' | |
| response = requests.post(identityURL, | |
| headers={'Content-Type': 'application/json','Accept':'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"methods":["password"],"password": | |
| {"user": | |
| {"domain": | |
| {"name":udomain}, | |
| "name":uname, | |
| "password": upassword | |
| }}}, | |
| "scope": | |
| { "project": | |
| {"id":uproject | |
| }}}}) | |
| return response.headers['X-Subject-Token'] | |
| def get_unscoped_token(uname,upassword,udomain,uregion): | |
| identityURL = 'https://identity.' + uregion + '.cloud.global.fujitsu.com/v3/auth/tokens' | |
| response = requests.post(identityURL, | |
| headers={'Content-Type': 'application/json','Accept':'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"methods":["password"],"password": | |
| {"user": | |
| {"domain": | |
| {"name":udomain}, | |
| "name":uname, | |
| "password": upassword | |
| }}}}}) | |
| return response.headers['X-Subject-Token'] | |
| # get a central identity portal token | |
| def get_unscoped_idtoken(uname,upassword,udomain): | |
| response = requests.post('https://auth-api.jp-east-1.paas.cloud.global.fujitsu.com/API/paas/auth/token', | |
| headers={'Content-Type': 'application/json'}, | |
| json={"auth": | |
| {"identity": | |
| {"password": | |
| {"user": | |
| {"contract_number":udomain, | |
| "name":uname, | |
| "password": upassword | |
| }}}}}) | |
| return response.headers['X-Access-Token'] | |
| # create a container | |
| def create_new_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| print scoped_k5token | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name | |
| print identityURL | |
| response = requests.put(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| def upload_file_to_container(adminUser,adminPassword,project,container_name,file_name,file_path,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| uploadfile = open(file_path, 'rb') | |
| data = uploadfile.read() | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| response = requests.put(identityURL, | |
| data=data, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/octet-stream'}) | |
| uploadfile.close | |
| return response | |
| def import_from_container_to_k5(adminUser,adminPassword,project,container_name,file_name,display_name,file_path,os_type,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| k5ContainerURL = '/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| image_id = str(uuid.uuid4()) | |
| encodedPassword = base64.b64encode(adminPassword) | |
| vmimportURL = 'https://vmimport.' + region + '.cloud.global.fujitsu.com/v1/imageimport' | |
| response = requests.post(vmimportURL, | |
| headers={'X-Auth-Token':scoped_k5token}, | |
| json={"name":display_name, | |
| "location":k5ContainerURL, | |
| "id":image_id, | |
| "conversion": True, | |
| "os_type":os_type, | |
| "user_name":adminUser, | |
| "password":encodedPassword, | |
| "domain_name":contract}) | |
| return response.json() | |
| def verify_image_import_status(adminUser,adminPassword,project,image_id,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| vmimportURL = 'https://vmimport.' + region + '.cloud.global.fujitsu.com/v1/imageimport/' + image_id + '/status' | |
| response = requests.get(vmimportURL, | |
| headers={'X-Auth-Token':scoped_k5token}) | |
| return response.json() | |
| def upload_manifest_to_container(adminUser,adminPassword,project,container_name,file_name,prefix,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/' + file_name | |
| response = requests.put(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'X-Object-Manifest': container_name + '/' + prefix}) | |
| return response | |
| # list items in a container | |
| def view_items_in_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '?format=json' | |
| response = requests.get(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| # download item in a container | |
| def download_item_in_storage_container(adminUser,adminPassword,project,container_name,contract,region): | |
| # get a regional domain scoped token to make queries to facilitate conversion of object names to ids | |
| scoped_k5token = get_scoped_token(adminUser,adminPassword,project,contract,region) | |
| identityURL = 'https://objectstorage.' + region + '.cloud.global.fujitsu.com/v1/AUTH_' + project + '/' + container_name + '/manifest' | |
| print identityURL | |
| response = requests.get(identityURL, | |
| headers={'X-Auth-Token':scoped_k5token,'Content-Type': 'application/json'}) | |
| return response | |
| def make_out_filename(prefix, idx): | |
| '''Make a filename with a serial number suffix.''' | |
| return prefix + str(idx).zfill(4) | |
| def bsplit(in_filename, bytes_per_file,os_type): | |
| '''Split the input file in_filename into output files of | |
| bytes_per_file bytes each. Last file may have less bytes.''' | |
| in_fil = open(in_filename, "rb") | |
| outfil_idx = 1 | |
| out_filename = make_out_filename(os_type, outfil_idx) | |
| out_fil = open(out_filename, "wb") | |
| byte_count = tot_byte_count = file_count = 0 | |
| c = in_fil.read(1) | |
| # Loop over the input and split it into multiple files | |
| # of bytes_per_file bytes each (except possibly for the | |
| # last file, which may have less bytes. | |
| while c != '': | |
| byte_count += 1 | |
| out_fil.write(c) | |
| # Bump vars; change to next output file. | |
| if byte_count >= bytes_per_file: | |
| tot_byte_count += byte_count | |
| byte_count = 0 | |
| file_count += 1 | |
| out_fil.close() | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,out_filename,out_filename,contract,region) | |
| print "Uploaded Package – " + str(file_count) | |
| os.remove(out_filename) | |
| outfil_idx += 1 | |
| out_filename = make_out_filename(os_type, outfil_idx) | |
| out_fil = open(out_filename, "wb") | |
| c = in_fil.read(1) | |
| # Clean up. | |
| in_fil.close() | |
| if not out_fil.closed: | |
| out_fil.close() | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,out_filename,out_filename,contract,region) | |
| print "\nUploaded Package – " + str(file_count) | |
| os.remove(out_filename) | |
| if byte_count == 0: | |
| os.remove(out_filename) | |
| # now create manifest file | |
| result = upload_manifest_to_container(adminUser,adminPassword,defaultid,container_name,file_name,os_type,contract,region) | |
| return result | |
| def main(): | |
| try: | |
| # ensure minimium commandline paramaters have been supplied | |
| if (len(sys.argv)<6): | |
| print("Usage1: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu | |
| sys.exit(2) | |
| # load the command line parameters | |
| myopts, args = getopt.getopt(sys.argv[1:],"i:c:n:p:t:s:",["imagepath=","container=","name=","projects=","type=","size="]) | |
| except getopt.GetoptError: | |
| # if the parameters are incorrect display error message | |
| print("Usage2: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu | |
| sys.exit(2) | |
| # define global variables from the command line parameters | |
| global container_name | |
| global display_name | |
| global bytes_per_file | |
| global os_type | |
| global file_path | |
| global file_name | |
| # set default chunk size for large images that needs to be broken up must be below 5GB for Swift Object Storage | |
| bytes_per_file = 1048576000 #5242880 #1048576000 #262144000 # 250Mb chunks | |
| ############################### | |
| # o == option | |
| # a == argument passed to the o | |
| ############################### | |
| for o, a in myopts: | |
| if o in ('-i','–imagepath'): | |
| file_path=a | |
| elif o in ('-c','–container'): | |
| container_name=a | |
| elif o in ('-n','–name'): | |
| display_name=a | |
| elif o in ('-p','–projects'): | |
| projects=a | |
| elif o in ('-t','–type'): | |
| os_type=a | |
| elif o in ('-s','–size'): | |
| bytes_per_file=int(a) | |
| else: | |
| print("Usage3: %s -i 'path_to_image' -c 'container_name' -n 'image_display_name' -p '{project1,project2,project3}' -t [ubuntu | |
| # extract filename from file path suplied at cli | |
| file_name = ntpath.basename(file_path) | |
| # attempt to read the contents of the container to see if it already exists | |
| result = view_items_in_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| # check to see if container already exists, if not then create it | |
| if (result.status_code == 404): | |
| # create container | |
| print "\nCreating new container : " + container_name | |
| result = create_new_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| print "\nCreated new container : " + container_name | |
| # check size of file to be uploaded is less than 250GB, if not split into smaller chunks for upload | |
| if (os.path.getsize(file_path) > bytes_per_file): | |
| # loop through image file for multi-part upload | |
| print "\n———- Starting multi-part file upload to K5 object storage —— \n" | |
| result = bsplit(file_path, bytes_per_file,os_type) | |
| print "\n———- Finished multi-part file upload to K5 object storage —— \n" | |
| else: | |
| # simple file upload to container | |
| print "\n———- Starting simple file upload to K5 object storage —— \n" | |
| result = upload_file_to_container(adminUser,adminPassword,defaultid,container_name,file_name,file_path,contract,region) | |
| print "\n———- Finished simple file upload to K5 object storage —— \n" | |
| # list container | |
| print "\n———- List container contents K5 object storage start —— \n" | |
| result = view_items_in_storage_container(adminUser,adminPassword,defaultid,container_name,contract,region) | |
| print result | |
| print "\n———- List container contents K5 object storage end —— \n" | |
| # Register image with K5 | |
| print "\n———- Registering image with K5 —— \n" | |
| result = import_from_container_to_k5(adminUser,adminPassword,defaultid,container_name,file_name,display_name,file_path,os_type,contract,region) | |
| image_id = result['import_id'] | |
| print result | |
| print "\n———- K5 Image import_id : " + image_id + "\n" | |
| # Get import status | |
| print "\n———- Check import status ———- \n\n" | |
| result = verify_image_import_status(adminUser,adminPassword,defaultid,image_id,contract,region) | |
| print result | |
| while ((result['import_status'] != "succeeded") and (result['import_status'] != "failed")): | |
| time.sleep(300) | |
| print "\n———- Check import status ———- \n" | |
| result = verify_image_import_status(adminUser,adminPassword,defaultid,image_id,contract,region) | |
| print result | |
| print "End of Import Process – Import status >>> " + result['import_status'] | |
| if __name__ == "__main__": | |
| main() | |
rawを表示
k5ImageUpload.py
❤️ GitHub によってホストされています
スクリプト出力例:
C:\Users\landg\>python K5ImageUpload.py -i "c:\Users\landg\Downloads\cirrosuploadtest.vmdk" -c uploaddemo12 -n "h
ello k5 milti image" -t ubuntu -p NotUsed -s 5242880
Creating new container : uploaddemo12
31c0f27e562c4b3089a546c175c144e4
https://objectstorage.uk-1.cloud.global.fujitsu.com/v1/AUTH_eadb882573ac40b1b101
eac93009a313/uploaddemo12
Created new container : uploaddemo12
---------- Starting multi-part file upload to K5 object storage ------
Uploaded Package - 1
Uploaded Package - 2
Uploaded Package - 3
Uploaded Package - 4
Uploaded Package - 4
---------- Finished multi-part file upload to K5 object storage ------
---------- List container contents K5 object storage start ------
<Response [200]>
---------- List container contents K5 object storage end ------
---------- Registering image with K5 ------
{u'import_id': u'6a0b58c5-bcda-4a64-919e-23f06b8338ad'}
---------- K5 Image import_id : 6a0b58c5-bcda-4a64-919e-23f06b8338ad
---------- Check import status ----------
{u'conversion': True, u'name': u'hello k5 milti image', u'container_format': u'b
are', u'min_ram': u'0', u'ovf_location': u'', u'disk_format': u'raw', u'domain_n
ame': u'YssmW1yI', u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uplo
addemo12/cirrosuploadtest.vmdk', u'min_disk': u'0', u'progress': u'0', u'os_type
': u'ubuntu', u'password': u'*', u'user_name': u'landg', u'id': u'70a38639-f819-
4375-b3d2-cfc99c2a148e', u'import_status': u'queued'}
---------- Check import status ----------
{u'conversion': True, u'name': u'hello k5 milti image', u'container_format': u'b
are', u'min_ram': u'0', u'ovf_location': u'', u'disk_format': u'raw', u'domain_n
ame': u'YssmW1yI', u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uplo
addemo12/cirrosuploadtest.vmdk', u'min_disk': u'0', u'progress': 0, u'os_type':
u'ubuntu', u'password': u'*', u'user_name': u'landg', u'id': u'70a38639-f819-437
5-b3d2-cfc99c2a148e', u'import_status': u'processing'}
---------- Check import status ----------
{u'container_format': u'bare', u'min_ram': 0, u'updated_at': u'2016-10-08T17:24:
38Z', u'file': u'/v2/images/70a38639-f819-4375-b3d2-cfc99c2a148e/file', u'owner'
: u'eadb882573ac40b1b101eac93009a313', u'id': u'70a38639-f819-4375-b3d2-cfc99c2a
148e', u'size': 41126400, u'conversion': True, u'self': u'/v2/images/70a38639-f8
19-4375-b3d2-cfc99c2a148e', u'disk_format': u'raw', u'domain_name': u'YssmW1yI',
u'location': u'/v1/AUTH_eadb882573ac40b1b101eac93009a313/uploaddemo12/cirrosupl
oadtest.vmdk', u'progress': 100, u'user_name': u'landg', u'schema': u'/v2/schema
s/image', u'status': u'active', u'import_status': u'succeeded', u'tags': [], u'v
isibility': u'private', u'BaseImageId': u'415b3a0a513aebc27d34c68bd8cdae8c', u'm
in_disk': 0, u'password': u'*', u'name': u'hello k5 milti image', u'created_at':
u'2016-10-08T17:24:32Z', u'ovf_location': u'', u'fcx.centos': u'true', u'protec
ted': False, u'os_type': u'ubuntu'}
End of Import Process - Import status >>> succeeded
Happy Stacking!
Originally published on allthingscloud.eu (2016-10-08).