diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1ed2455
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,144 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+*profiling_
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+*.out
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/en/_build/
+docs/zh_cn/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+.vscode
+.idea
+.DS_Store
+
+# custom
+*.pkl
+*.pkl.json
+*.log.json
+docs/modelzoo_statistics.md
+mmdet/.mim
+work_dirs/
+logs/
+ckpts/
+batchscript-*
+compare_2_profiling_data/
+llm_logs/
+aim_logs/
+nvmelogs/
+run_backup/
+runs/
+runs_bak/
+LLM_ALERT
+small_demo/
+7b_llama_nopp/
+
+# Pytorch
+*.pth
+*.py~
+*.sh~
+
+# Core
+core.*
+
+# Run
+llm_ckpts
\ No newline at end of file
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..0e8d65a
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "third_party/flash-attention"]
+	path = third_party/flash-attention
+	url = https://github.com/HazyResearch/flash-attention.git
+[submodule "third_party/apex"]
+	path = third_party/apex
+	url = https://github.com/NVIDIA/apex
diff --git a/README-zh-Hans.md b/README-zh-Hans.md
index ef37638..38a113b 100644
--- a/README-zh-Hans.md
+++ b/README-zh-Hans.md
@@ -98,7 +98,7 @@ streamlit run web_demo.py
 ```
 效果如下 
 
-![效果](https://github.com/InternLM/InternLM/assets/9102141/9658904b-09cc-4bbc-bfbf-27f43f24475b)
+![效果](https://github.com/InternLM/InternLM/assets/9102141/11b60ee0-47e4-42c0-8278-3051b2f17fe4)
 
 ### 基于InternLM高性能部署
 
diff --git a/third_party/apex b/third_party/apex
new file mode 160000
index 0000000..8ffc901
--- /dev/null
+++ b/third_party/apex
@@ -0,0 +1 @@
+Subproject commit 8ffc901e50bbf740fdb6d5bccb17f66a6ec8604e
diff --git a/third_party/flash-attention b/third_party/flash-attention
new file mode 160000
index 0000000..d2f4324
--- /dev/null
+++ b/third_party/flash-attention
@@ -0,0 +1 @@
+Subproject commit d2f4324f4c56e017fbf22dc421943793a8ca6c3b