github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

storage

pull/520/head
lijiaxing 2023-12-01 14:08:45 +08:00
parent b7f721fffb
commit 03e53871a7
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internlm/utils/storage_manager.py
View File

@ -569,7 +569,7 @@ class VolcClient(StorageClient):
upload_id = multi_result.upload_id upload_id = multi_result.upload_id
parts = [] parts = []
# 上传分片数据 # Upload shard data
with open(local_nvme_path, "rb") as f: with open(local_nvme_path, "rb") as f:
part_number = 1 part_number = 1
offset = 0 offset = 0
@ -586,7 +586,7 @@ class VolcClient(StorageClient):
offset += num_to_upload offset += num_to_upload
part_number += 1 part_number += 1
# 完成分片上传任务 # Complete the multipart upload task
handler.client.complete_multipart_upload(bucket_name, fp, upload_id, parts) handler.client.complete_multipart_upload(bucket_name, fp, upload_id, parts)
except handler.handler.exceptions.TosClientError as exc: except handler.handler.exceptions.TosClientError as exc:
@ -703,7 +703,8 @@ class AliClient(StorageClient):
offset = 0 offset = 0
while offset < total_size: while offset < total_size:
num_to_upload = min(part_size, total_size - offset) num_to_upload = min(part_size, total_size - offset)
# 调用SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象，重新计算起始追加位置。 # Calling the SizedFileAdapter method will generate a new file object
# and recalculate the starting append position.
result = handler.client.upload_part( result = handler.client.upload_part(
fp, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload) fp, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)
) )