From 307c4741d1de14f3f7e8fbbebbf5ab70e2075dd7 Mon Sep 17 00:00:00 2001
From: ytxiong <45058324+yingtongxiong@users.noreply.github.com>
Date: Tue, 1 Aug 2023 16:03:06 +0800
Subject: [PATCH] fix(initialize/launch.py): set default value for
 use_flash_attn (#158)

* add default for use_flash_attn

* fix lint
---
 internlm/initialize/launch.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/internlm/initialize/launch.py b/internlm/initialize/launch.py
index d2d61b1..276dcad 100644
--- a/internlm/initialize/launch.py
+++ b/internlm/initialize/launch.py
@@ -173,6 +173,10 @@ def args_sanity_check():
         logger.info("+" * 15 + " beta2_scheduler Info " + "+" * 15)  # pylint: disable=W1201
         logger.info(f"beta2_scheduler: {gpc.config.beta2_scheduler}")
 
+    # process the model config
+    if "use_flash_attn" not in gpc.config.model:
+        gpc.config.model._add_item("use_flash_attn", True)
+
 
 def launch(
     config: Union[str, Path, Config, Dict],