From 93627e4f9dd6468c68e8d9ae218d64d1f1bf1758 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 15 Aug 2024 16:01:48 +0800 Subject: [PATCH] perf: clean site packages --- Dockerfile-base | 4 +- poetry.lock | 282 +---------------------------------- pyproject.toml | 2 - receptor | 168 --------------------- utils/clean_site_packages.sh | 36 +++++ 5 files changed, 40 insertions(+), 452 deletions(-) delete mode 100755 receptor create mode 100644 utils/clean_site_packages.sh diff --git a/Dockerfile-base b/Dockerfile-base index 067cad40e..732798d8b 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -46,9 +46,11 @@ ARG PIP_MIRROR=https://pypi.org/simple RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=core \ --mount=type=bind,source=poetry.lock,target=poetry.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=utils/clean_site_packages.sh,target=clean_site_packages.sh \ set -ex \ && python3 -m venv /opt/py3 \ && pip install poetry -i ${PIP_MIRROR} \ && poetry config virtualenvs.create false \ && . /opt/py3/bin/activate \ - && poetry install --only main + && poetry install --only main \ + && bash clean_site_packages.sh diff --git a/poetry.lock b/poetry.lock index 04bd3f7e7..dd4ac20dc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -539,22 +539,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "appnope" -version = "0.1.4" -description = "Disable App Nap on macOS >= 10.9" -optional = false -python-versions = ">=3.6" -files = [ - {file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"}, - {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, -] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "asgiref" version = "3.8.1" @@ -590,29 +574,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "asttokens" -version = "2.4.1" -description = "Annotate AST trees with source code positions" -optional = false -python-versions = "*" -files = [ - {file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"}, - {file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"}, -] - -[package.dependencies] -six = ">=1.12.0" - -[package.extras] -astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"] -test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "async-timeout" version = "4.0.3" @@ -879,22 +840,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "backcall" -version = "0.2.0" -description = "Specifications for callback functions passed in to an API" -optional = false -python-versions = "*" -files = [ - {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, - {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, -] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "bce-python-sdk" version = "0.8.87" @@ -2549,25 +2494,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "executing" -version = "2.0.1" -description = "Get the currently executing AST node of a frame, and other information" -optional = false -python-versions = ">=3.5" -files = [ - {file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"}, - {file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"}, -] - -[package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "fido2" version = "1.1.3" @@ -3349,49 +3275,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "ipython" -version = "8.14.0" -description = "IPython: Productive Interactive Computing" -optional = false -python-versions = ">=3.9" -files = [ - {file = "ipython-8.14.0-py3-none-any.whl", hash = "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"}, - {file = "ipython-8.14.0.tar.gz", hash = "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1"}, -] - -[package.dependencies] -appnope = {version = "*", markers = "sys_platform == \"darwin\""} -backcall = "*" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -decorator = "*" -jedi = ">=0.16" -matplotlib-inline = "*" -pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} -pickleshare = "*" -prompt-toolkit = ">=3.0.30,<3.0.37 || >3.0.37,<3.1.0" -pygments = ">=2.4.0" -stack-data = "*" -traitlets = ">=5" - -[package.extras] -all = ["black", "curio", "docrepr", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.21)", "pandas", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] -black = ["black"] -doc = ["docrepr", "ipykernel", "matplotlib", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"] -kernel = ["ipykernel"] -nbconvert = ["nbconvert"] -nbformat = ["nbformat"] -notebook = ["ipywidgets", "notebook"] -parallel = ["ipyparallel"] -qtconsole = ["qtconsole"] -test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] -test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "iso8601" version = "2.1.0" @@ -3459,30 +3342,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "jedi" -version = "0.19.1" -description = "An autocompletion tool for Python that can be used for text editors." -optional = false -python-versions = ">=3.6" -files = [ - {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, - {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, -] - -[package.dependencies] -parso = ">=0.8.3,<0.9.0" - -[package.extras] -docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "jinja2" version = "3.1.2" @@ -3972,25 +3831,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "matplotlib-inline" -version = "0.1.7" -description = "Inline Matplotlib backend for Jupyter" -optional = false -python-versions = ">=3.8" -files = [ - {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, - {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, -] - -[package.dependencies] -traitlets = "*" - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "maxminddb" version = "2.6.2" @@ -4768,26 +4608,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "parso" -version = "0.8.4" -description = "A Python Parser" -optional = false -python-versions = ">=3.6" -files = [ - {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, - {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, -] - -[package.extras] -qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["docopt", "pytest"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "passlib" version = "1.7.4" @@ -4861,22 +4681,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "pickleshare" -version = "0.7.5" -description = "Tiny 'shelve'-like database with concurrency support" -optional = false -python-versions = "*" -files = [ - {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, - {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, -] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "pillow" version = "10.0.1" @@ -5219,25 +5023,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "pure-eval" -version = "0.2.2" -description = "Safely evaluate AST nodes without side effects" -optional = false -python-versions = "*" -files = [ - {file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"}, - {file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"}, -] - -[package.extras] -tests = ["pytest"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "pyasn1" version = "0.5.0" @@ -6429,27 +6214,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "receptorctl" -version = "1.4.8" -description = "\"Receptorctl is a front-end CLI and importable Python library that interacts with Receptor over its control socket interface.\"" -optional = false -python-versions = "*" -files = [ - {file = "receptorctl-1.4.8-py3-none-any.whl", hash = "sha256:468d949f85bb50de9fd77cf5af2c173450b346fada787e3907a16f161aff17d6"}, - {file = "receptorctl-1.4.8.tar.gz", hash = "sha256:5cf94bec214641506f8f956247c40cb8cc73e84b1a2490a125e49d35159dfa62"}, -] - -[package.dependencies] -click = "*" -python-dateutil = "*" -pyyaml = "*" - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "redis" version = "5.0.3" @@ -6968,30 +6732,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "stack-data" -version = "0.6.3" -description = "Extract data from python stack frames and tracebacks for informative displays" -optional = false -python-versions = "*" -files = [ - {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, - {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, -] - -[package.dependencies] -asttokens = ">=2.1.0" -executing = ">=1.2.0" -pure-eval = "*" - -[package.extras] -tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "stevedore" version = "5.2.0" @@ -7115,26 +6855,6 @@ type = "legacy" url = "https://mirrors.aliyun.com/pypi/simple" reference = "aliyun" -[[package]] -name = "traitlets" -version = "5.14.3" -description = "Traitlets Python configuration system" -optional = false -python-versions = ">=3.8" -files = [ - {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, - {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, -] - -[package.extras] -docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] -test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] - -[package.source] -type = "legacy" -url = "https://mirrors.aliyun.com/pypi/simple" -reference = "aliyun" - [[package]] name = "treelib" version = "1.6.4" @@ -7981,4 +7701,4 @@ reference = "aliyun" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "54e8ab1d59a70cc6857323145725f13f0ae48eb5f286683bb7d0c7bebb31fb45" +content-hash = "2dc429e66e78ab1e264e26da60df6a67cb8d5e501d80297332d673646bc0cf13" diff --git a/pyproject.toml b/pyproject.toml index 518609273..f0ad19047 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,7 +135,6 @@ pyopenssl = "23.2.0" redis = { url = "https://github.com/jumpserver-dev/redis-py/archive/refs/tags/v5.0.3.zip" } pymongo = "4.4.1" pyfreerdp = "0.0.2" -ipython = "8.14.0" forgerypy3 = "0.3.1" django-debug-toolbar = "4.1.0" pympler = "1.0.1" @@ -156,7 +155,6 @@ xlsxwriter = "^3.1.9" exchangelib = "^5.1.0" xmlsec = "^1.3.13" lxml = "5.2.1" -receptorctl = "^1.4.5" pydantic = "^2.7.4" annotated-types = "^0.6.0" httpx = "^0.27.0" diff --git a/receptor b/receptor deleted file mode 100755 index cd1e54d16..000000000 --- a/receptor +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env python3 -# coding: utf-8 - -import argparse -import logging -import os -import shutil -import signal -import subprocess -import tempfile - -from apps.libs.process.ssh import kill_ansible_ssh_process - -ANSIBLE_RUNNER_COMMAND = "ansible-runner" - -PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) -APPS_DIR = os.path.join(PROJECT_DIR, 'apps') -TEMP_DIR = os.path.join(PROJECT_DIR, "tmp") - -DEFAULT_SHARE_DIR = os.path.join(PROJECT_DIR, "data", "share") -DEFAULT_ANSIBLE_MODULES_DIR = os.path.join(APPS_DIR, "libs", "ansible", "modules") -DEFAULT_CONTROL_SOCK_PATH = os.path.join(DEFAULT_SHARE_DIR, "control.sock") - -DEFAULT_TCP_LISTEN_ADDRESS = "0.0.0.0:7521" - -logger = logging.getLogger(__name__) - -os.chdir(APPS_DIR) - - -class ReceptorService: - def __init__(self): - self.pid_file = os.path.join(TEMP_DIR, "receptor.pid") - self.receptor_command = [ - 'receptor', - '--local-only', - '--node', 'id=primary', - '--log-level', 'level=Error', - '--control-service', - 'service=control', - 'tcplisten={}'.format(DEFAULT_TCP_LISTEN_ADDRESS), - '--work-command', - 'worktype={}'.format(ANSIBLE_RUNNER_COMMAND), - 'command={}'.format(ANSIBLE_RUNNER_COMMAND), - 'params=worker', - 'allowruntimeparams=true', - '--work-command', - 'worktype={}'.format("kill"), - 'command={}'.format("python"), - "params={} kill".format(os.path.join(PROJECT_DIR, "receptor")), - 'allowruntimeparams=true' - ] - - - @staticmethod - def before_start(): - os.makedirs(os.path.join(DEFAULT_SHARE_DIR), exist_ok=True) - status_dir = os.path.join(tempfile.gettempdir(), "receptor") - if os.path.exists(status_dir): - shutil.rmtree(status_dir) - - def start(self): - self.before_start() - if os.path.exists(self.pid_file): - with open(self.pid_file, 'r') as f: - pid_str = f.read() - try: - pid = int(pid_str) - os.kill(pid, 0) - print("\n- Receptor service is already running.") - return - except ProcessLookupError: - print("\n- PID file exists but process does not, starting Receptor...") - except ValueError: - print("\n- PID file is corrupted, starting Receptor...") - os.remove(self.pid_file) - - os.environ.setdefault('ANSIBLE_LIBRARY', DEFAULT_ANSIBLE_MODULES_DIR) - os.environ.update({'PYTHONPATH': APPS_DIR}) - process = subprocess.Popen(self.receptor_command) - with open(self.pid_file, 'w') as f: - f.write(str(process.pid)) - print("\n- Receptor service started successfully.") - - def exit_handler(signum, frame): - process.terminate() - process.kill() - - signal.signal(signal.SIGINT, exit_handler) - signal.signal(signal.SIGTERM, exit_handler) - process.wait() - - def stop(self): - if not os.path.exists(self.pid_file): - print("\n- Receptor service is not running.") - return - with open(self.pid_file, 'r') as f: - pid = int(f.read()) - try: - os.kill(pid, signal.SIGTERM) - os.remove(self.pid_file) - print("\n- Receptor service stopped successfully.") - except ProcessLookupError: - print("\n- Failed to stop Receptor service: Process does not exist.") - os.remove(self.pid_file) - - def restart(self): - self.stop() - self.start() - - def status(self): - if os.path.exists(self.pid_file): - with open(self.pid_file, 'r') as f: - pid_str = f.read() - try: - pid = int(pid_str) - os.kill(pid, 0) - print("\n- Receptor service is running.") - return - except ProcessLookupError: - print("\n- Receptor service is not running.") - else: - print("\n- Receptor service is not running.") - - -def handle_receptor_action(args): - action = args.action - srv = ReceptorService() - if action == "start": - srv.start() - elif action == 'stop': - srv.stop() - elif action == "restart": - srv.restart() - elif action == "status": - srv.status() - elif action == "kill": - kill_progress_tree() - - -def kill_progress_tree(pid=None): - if not pid: - try: - pid_input = input() - pid = int(pid_input) - logger.info("progress {} will be kill".format(pid)) - kill_ansible_ssh_process(pid) - except Exception as e: - logger.error(e) - return - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=""" - Jumpserver receptor service control tools; - """ - ) - parser.add_argument( - 'action', type=str, - choices=("start", "stop", "restart", "status", "kill"), - help="Action to run" - ) - - # parser.add_argument('--pid', type=int, default=42, help='what PID you want to kill') - - args = parser.parse_args() - handle_receptor_action(args) diff --git a/utils/clean_site_packages.sh b/utils/clean_site_packages.sh new file mode 100644 index 000000000..47f7358ba --- /dev/null +++ b/utils/clean_site_packages.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# + +lib_path="/opt/py3/lib/python3.11/site-packages" + +# 清理不需要的模块 +need_clean="jedi botocore/data" +for i in $need_clean; do + rm -rf "${lib_path}/${i}" +done + +# 清理 ansible connection 中 不需要的模块 +ansible_connection="${lib_path}/ansible_collections" +need_clean="fortinet dellemc f5networks netapp theforeman google azure cyberark ibm + netbox purestorage inspur netapp_eseries sensu check_point vyos arista" +for i in $need_clean; do + echo "rm -rf ${ansible_connection:-tmp}/${i}" + rm -rf "${ansible_connection:-tmp}/${i}" +done + +# 清理缓存文件 +cd lib_path +find . -name "*.pyc" -exec rm -f {} \; + +# 清理不需要的国际化文件 +find . -name 'locale' -o -name 'locales' -type d | while read -r dir; do + find "$dir" -mindepth 1 -maxdepth 1 -type d \ + ! -name 'zh_Hans' \ + ! -name 'zh_Hant' \ + ! -name 'zh_CN' \ + ! -name 'en' \ + ! -name 'en_US' \ + ! -name 'ja' \ + ! -name 'fr' \ + -exec rm -rf {} \; +done