github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

storage

pull/527/head
lijiaxing 2023-12-06 15:15:54 +08:00
parent a34c31c08e
commit ff62cf2a7c
1 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
internlm/utils/storage_manager.py
View File

@ -340,24 +340,24 @@ class Boto3Client(StorageClient):
super().__init__(boto3) super().__init__(boto3)
self.botocore = botocore self.botocore = botocore
try: try:
if os.environ.get["S3_ACCESS_KEY_ID"] is not None and os.environ.get["ACCESS_KEY"] is not None: if os.environ.get("S3_ACCESS_KEY_ID") is not None and os.environ.get("ACCESS_KEY") is not None:
s3_access_key_id = os.environ["ACCESS_KEY"] s3_access_key_id = os.environ["ACCESS_KEY"]
logger.warning("Both 'S3_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default") logger.warning("Both 'S3_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default")
elif os.environ.get["ACCESS_KEY"] is None: elif os.environ.get("ACCESS_KEY") is None:
s3_access_key_id = os.environ["S3_ACCESS_KEY_ID"] s3_access_key_id = os.environ["S3_ACCESS_KEY_ID"]
else: else:
s3_access_key_id = os.environ["ACCESS_KEY"] s3_access_key_id = os.environ["ACCESS_KEY"]
if ( if (
os.environ.get["S3_SECRET_ACCESS_KEY_ID"] is not None os.environ.get("S3_SECRET_ACCESS_KEY_ID") is not None
and os.environ.get["SECRET_ACCESS_KEY"] is not None and os.environ.get("SECRET_ACCESS_KEY") is not None
): ):
s3_secret_access_key = os.environ["SECRET_ACCESS_KEY"] s3_secret_access_key = os.environ["SECRET_ACCESS_KEY"]
logger.warning( logger.warning(
"Both 'S3_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, " "Both 'S3_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, "
"'SECRET_ACCESS_KEY' will be used by default" "'SECRET_ACCESS_KEY' will be used by default"
) )
elif os.environ.get["SECRET_ACCESS_KEY"] is None: elif os.environ.get("SECRET_ACCESS_KEY") is None:
s3_secret_access_key = os.environ["S3_SECRET_ACCESS_KEY_ID"] s3_secret_access_key = os.environ["S3_SECRET_ACCESS_KEY_ID"]
else: else:
s3_secret_access_key = os.environ["SECRET_ACCESS_KEY"] s3_secret_access_key = os.environ["SECRET_ACCESS_KEY"]
@ -485,24 +485,24 @@ class VolcClient(StorageClient):
super().__init__(tos) super().__init__(tos)
try: try:
if os.environ.get["VOLC_ACCESS_KEY_ID"] is not None and os.environ.get["ACCESS_KEY"] is not None: if os.environ.get("VOLC_ACCESS_KEY_ID") is not None and os.environ.get("ACCESS_KEY") is not None:
access_key = os.environ["ACCESS_KEY"] access_key = os.environ["ACCESS_KEY"]
logger.warning("Both 'VOLC_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default") logger.warning("Both 'VOLC_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default")
elif os.environ.get["ACCESS_KEY"] is None: elif os.environ.get("ACCESS_KEY") is None:
access_key = os.environ["VOLC_ACCESS_KEY_ID"] access_key = os.environ["VOLC_ACCESS_KEY_ID"]
else: else:
access_key = os.environ["ACCESS_KEY"] access_key = os.environ["ACCESS_KEY"]
if ( if (
os.environ.get["VOLC_SECRET_ACCESS_KEY_ID"] is not None os.environ.get("VOLC_SECRET_ACCESS_KEY_ID") is not None
and os.environ.get["SECRET_ACCESS_KEY"] is not None and os.environ.get("SECRET_ACCESS_KEY") is not None
): ):
secret_key = os.environ["SECRET_ACCESS_KEY"] secret_key = os.environ["SECRET_ACCESS_KEY"]
logger.warning( logger.warning(
"Both 'VOLC_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, " "Both 'VOLC_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, "
"'SECRET_ACCESS_KEY' will be used by default" "'SECRET_ACCESS_KEY' will be used by default"
) )
elif os.environ.get["SECRET_ACCESS_KEY"] is None: elif os.environ.get("SECRET_ACCESS_KEY") is None:
secret_key = os.environ["VOLC_SECRET_ACCESS_KEY_ID"] secret_key = os.environ["VOLC_SECRET_ACCESS_KEY_ID"]
else: else:
secret_key = os.environ["SECRET_ACCESS_KEY"] secret_key = os.environ["SECRET_ACCESS_KEY"]
@ -670,24 +670,24 @@ class AliClient(StorageClient):
super().__init__(oss2) super().__init__(oss2)
try: try:
if os.environ.get["ALI_ACCESS_KEY_ID"] is not None and os.environ.get["ACCESS_KEY"] is not None: if os.environ.get("ALI_ACCESS_KEY_ID") is not None and os.environ.get("ACCESS_KEY") is not None:
access_key = os.environ["ACCESS_KEY"] access_key = os.environ["ACCESS_KEY"]
logger.warning("Both 'ALI_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default") logger.warning("Both 'ALI_ACCESS_KEY_ID' and 'ACCESS_KEY' exist, 'ACCESS_KEY' will be used by default")
elif os.environ.get["ACCESS_KEY"] is None: elif os.environ.get("ACCESS_KEY") is None:
access_key = os.environ["ALI_ACCESS_KEY_ID"] access_key = os.environ["ALI_ACCESS_KEY_ID"]
else: else:
access_key = os.environ["ACCESS_KEY"] access_key = os.environ["ACCESS_KEY"]
if ( if (
os.environ.get["ALI_SECRET_ACCESS_KEY_ID"] is not None os.environ.get("ALI_SECRET_ACCESS_KEY_ID") is not None
and os.environ.get["SECRET_ACCESS_KEY"] is not None and os.environ.get("SECRET_ACCESS_KEY") is not None
): ):
secret_key = os.environ["SECRET_ACCESS_KEY"] secret_key = os.environ["SECRET_ACCESS_KEY"]
logger.warning( logger.warning(
"Both 'ALI_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, " "Both 'ALI_SECRET_ACCESS_KEY_ID' and 'SECRET_ACCESS_KEY' exist, "
"'SECRET_ACCESS_KEY' will be used by default" "'SECRET_ACCESS_KEY' will be used by default"
) )
elif os.environ.get["SECRET_ACCESS_KEY"] is None: elif os.environ.get("SECRET_ACCESS_KEY") is None:
secret_key = os.environ["ALI_SECRET_ACCESS_KEY_ID"] secret_key = os.environ["ALI_SECRET_ACCESS_KEY_ID"]
else: else:
secret_key = os.environ["SECRET_ACCESS_KEY"] secret_key = os.environ["SECRET_ACCESS_KEY"]