8000 Merge pull request #419 from qiniu/features/add-upload-metadata · qiniu/python-sdk@0767741 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0767741

Browse files
lyyyunalyyyuna
authored
Merge pull request #419 from qiniu/features/add-upload-metadata
upload can add metadata
2 parents 852a32e + af1485c commit 0767741

File tree

2 files changed

+101
-19
lines changed

2 files changed

+101
-19
lines changed

‎qiniu/services/storage/uploader.py

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
def put_data(
1414
up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None,
15-
fname=None, hostscache_dir=None):
15+
fname=None, hostscache_dir=None, metadata=None):
1616
"""上传二进制流到七牛
1717
1818
Args:
@@ -23,7 +23,8 @@ def put_data(
2323
mime_type: 上传数据的mimeType
2424
check_crc: 是否校验crc32
2525
progress_handler: 上传进度
26-
hostscache_dir： host请求 缓存文件保存位置
26+
hostscache_dir: host请求 缓存文件保存位置
27+
metadata: 元数据
2728
2829
Returns:
2930
一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
@@ -41,13 +42,14 @@ def put_data(
4142
final_data = data
4243

4344
crc = crc32(final_data)
44-
return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname)
45+
return _form_put(up_token, key, final_data, params, mime_type,
46+
crc, hostscache_dir, progress_handler, fname, metadata=metadata)
4547

4648

4749
def put_file(up_token, key, file_path, params=None,
4850
mime_type='application/octet-stream', check_crc=False,
4951
progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None,
50-
part_size=None, version=None, bucket_name=None):
52+
part_size=None, version=None, bucket_name=None, metadata=None):
5153
"""上传文件到七牛
5254
5355
Args:
@@ -59,10 +61,11 @@ def put_file(up_token, key, file_path, params=None,
5961
check_crc: 是否校验crc32
6062
progress_handler: 上传进度
6163
upload_progress_recorder: 记录上传进度，用于断点续传
62-
hostscache_dir： host请求 缓存文件保存位置
63-
version 分片上传版本 目前支持v1/v2版本 默认v1
64-
part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB
65-
bucket_name 分片上传v2字段 空间名称
64+
hostscache_dir: host请求 缓存文件保存位置
65+
version: 分片上传版本 目前支持v1/v2版本 默认v1
66+
part_size: 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB
67+
bucket_name: 分片上传v2字段 空间名称
68+
metadata: 元数据信息
6669
6770
Returns:
6871
一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
@@ -78,18 +81,17 @@ def put_file(up_token, key, file_path, params=None,
7881
mime_type, progress_handler,
7982
upload_progress_recorder=upload_progress_recorder,
8083
modify_time=modify_time, keep_last_modified=keep_last_modified,
81-
part_size=part_size, version=version, bucket_name=bucket_name)
84+
part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata)
8285
else:
8386
crc = file_crc32(file_path)
8487
ret, info = _form_put(up_token, key, input_stream, params, mime_type,
8588
crc, hostscache_dir, progress_handler, file_name,
86-
modify_time=modify_time, keep_last_modified=keep_last_modified)
89+
modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata)
8790
return ret, info
8891

8992

9093
def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None,
91-
modify_time=None,
92-
keep_last_modified=False):
94+
modify_time=None, keep_last_modified=False, metadata=None):
9395
fields = {}
9496
if params:
9597
for k, v in params.items():
@@ -114,6 +116,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None,
114116
if modify_time and keep_last_modified:
115117
fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time)
116118

119+
if metadata:
120+
for k, v in metadata.items():
121+
if k.startswith('x-qn-meta-'):
122+
fields[k] = str(v)
123+
117124
r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)})
118125
if r is None and info.need_retry():
119126
if info.connect_failed:
@@ -135,10 +142,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None,
135142
def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None,
136143
mime_type=None, progress_handler=None,
137144
upload_progress_recorder=None, modify_time=None, keep_last_modified=False,
138-
part_size=None, version=None, bucket_name=None):
145+
part_size=None, version=None, bucket_name=None, metadata=None):
139146
task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type,
140147
progress_handler, upload_progress_recorder, modify_time, keep_last_modified,
141-
part_size, version, bucket_name)
148+
part_size, version, bucket_name, metadata)
142149
return task.upload()
143150

144151

@@ -167,7 +174,7 @@ class _Resume(object):
167174

168175
def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type,
169176
progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None,
170-
version=None, bucket_name=None):
177+
version=None, bucket_name=None, metadata=None):
171178
"""初始化断点续上传"""
172179
self.up_token = up_token
173180
self.key = key
@@ -184,6 +191,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache
184191
self.version = version or 'v1'
185192
self.part_size = part_size or config._BLOCK_SIZE
186193
self.bucket_name = bucket_name
194+
self.metadata = metadata
187195

188196
def record_upload_progress(self, offset):
189197
record_data = {
@@ -294,9 +302,9 @@ def upload(self):
294302
elif self.version == 'v2':
295303
make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId
296304
return self.make_file_v2(self.blockStatus, make_file_url, self.file_name,
297-
self.mime_type, self.params)
305+
self.mime_type, self.params, self.metadata)
298306

299-
def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None):
307+
def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None):
300308
"""completeMultipartUpload"""
301309
parts = self.get_parts(block_status)
302310
headers = {
@@ -306,7 +314,8 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom
306314
'parts': parts,
307315
'fname': file_name,
308316
'mimeType': mime_type,
309-
'customVars': customVars
317+
'customVars': customVars,
318+
'metadata': metadata
310319
}
311320
ret, info = self.post_with_headers(url, json.dumps(data), headers=headers)
312321
if ret is not None and ret != {}:
@@ -354,12 +363,17 @@ def file_url(self, host):
354363
if self.params:
355364
for k, v in self.params.items():
356365
url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v)))
357-
pass
358366

359367
if self.modify_time and self.keep_last_modified:
360368
url.append(
361369
"x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time))))
362370

371+
if self.metadata:
372+
for k, v in self.metadata.items():
373+
if k.startswith('x-qn-meta-'):
374+
url.append(
375+
"{0}/{1}".format(k, urlsafe_base64_encode(v)))
376+
363377
url = '/'.join(url)
364378
return url
365379

‎test_qiniu.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,12 @@ def test_set_object_lifecycle_with_cond(self):
498498
class UploaderTestCase(unittest.TestCase):
499499
mime_type = "text/plain"
500500
params = {'x:a': 'a'}
501+
metadata = {
502+
'x-qn-meta-name': 'qiniu',
503+
'x-qn-meta-age': '18'
504+
}
501505
q = Auth(access_key, secret_key)
506+
bucket = BucketManager(q)
502507

503508
def test_put(self):
504509
key = 'a\\b\\c"hello'
@@ -594,11 +599,40 @@ def test_putData_without_fname2(self):
594599
print(info)
595600
assert ret is not None
596601

602+
def test_put_file_with_metadata(self):
603+
localfile = __file__
604+
key = 'test_file_with_metadata'
605+
606+
token = self.q.upload_token(bucket_name, key)
607+
ret, info = put_file(token, key, localfile, metadata=self.metadata)
608+
assert ret['key'] == key
609+
assert ret['hash'] == etag(localfile)
610+
ret, info = self.bucket.stat(bucket_name, key)
611+
assert 'x-qn-meta' in ret
612+
assert ret['x-qn-meta']['name'] == 'qiniu'
613+
assert ret['x-qn-meta']['age'] == '18'
614+
615+
def test_put_data_with_metadata(self):
616+
key = 'put_data_with_metadata'
617+
data = 'hello metadata!'
618+
token = self.q.upload_token(bucket_name, key)
619+
ret, info = put_data(token, key, data, metadata=self.metadata)
620+
assert ret['key'] == key
621+
ret, info = self.bucket.stat(bucket_name, key)
622+
assert 'x-qn-meta' in ret
623+
assert ret['x-qn-meta']['name'] == 'qiniu'
624+
assert ret['x-qn-meta']['age'] == '18'
625+
597626

598627
class ResumableUploaderTestCase(unittest.TestCase):
599628
mime_type = "text/plain"
600629
params = {'x:a': 'a'}
630+
metadata = {
631+
'x-qn-meta-name': 'qiniu',
632+
'x-qn-meta-age': '18'
633+
}
601634
q = Auth(access_key, secret_key)
635+
bucket = BucketManager(q)
602636

603637
def test_put_stream(self):
604638
localfile = __file__
@@ -715,6 +749,40 @@ def test_put_stream_with_key_limits(self):
715749
self.mime_type)
716750
assert info.status_code == 200
717751

752+
def test_put_stream_with_metadata(self):
753+
localfile = __file__
754+
key = 'test_put_stream_with_metadata'
755+
size = os.stat(localfile).st_size
756+
set_default(default_zone=Zone('https://upload.qiniup.com'))
757+
with open(localfile, 'rb') as input_stream:
758+
token = self.q.upload_token(bucket_name, key)
759+
ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir,
760+
self.params, self.mime_type,
761+
part_size=None, version=None, bucket_name=None, metadata=self.metadata)
762+
assert ret['key'] == key
763+
ret, info = self.bucket.stat(bucket_name, key)
764+
assert 'x-qn-meta' in ret
765+
assert ret['x-qn-meta']['name'] == 'qiniu'
766+
assert ret['x-qn-meta']['age'] == '18'
767+
768+
def test_put_stream_v2_with_metadata(self):
769+
part_size = 1024 * 1024 * 4
770+
localfile = create_temp_file(part_size + 1)
771+
key = 'test_put_stream_v2_with_metadata'
772+
size = os.stat(localfile).st_size
773+
set_default(default_zone=Zone('https://upload.qiniup.com'))
774+
with open(localfile, 'rb') as input_stream:
775+
token = self.q.upload_token(bucket_name, key)
776+
ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir,
777+
self.params, self.mime_type,
778+
part_size=part_size, version='v2', bucket_name=bucket_name, metadata=self.metadata)
779+
assert ret['key'] == key
780+
remove_temp_file(localfile)
781+
ret, info = self.bucket.stat(bucket_name, key)
782+
assert 'x-qn-meta' in ret
783+
assert ret['x-qn-meta']['name'] == 'qiniu'
784+
assert ret['x-qn-meta']['age'] == '18'
785+
718786

719787
class DownloadTestCase(unittest.TestCase):
720788
q = Auth(access_key, secret_key)

0 commit comments

Comments
 (0)
0