Browse Source

perf: clean site packages

pull/13998/head
ibuler 3 months ago committed by Bryan
parent
commit
93627e4f9d
  1. 4
      Dockerfile-base
  2. 282
      poetry.lock
  3. 2
      pyproject.toml
  4. 168
      receptor
  5. 36
      utils/clean_site_packages.sh

4
Dockerfile-base

@ -46,9 +46,11 @@ ARG PIP_MIRROR=https://pypi.org/simple
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=core \
--mount=type=bind,source=poetry.lock,target=poetry.lock \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=utils/clean_site_packages.sh,target=clean_site_packages.sh \
set -ex \
&& python3 -m venv /opt/py3 \
&& pip install poetry -i ${PIP_MIRROR} \
&& poetry config virtualenvs.create false \
&& . /opt/py3/bin/activate \
&& poetry install --only main
&& poetry install --only main \
&& bash clean_site_packages.sh

282
poetry.lock

@ -539,22 +539,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "appnope"
version = "0.1.4"
description = "Disable App Nap on macOS >= 10.9"
optional = false
python-versions = ">=3.6"
files = [
{file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"},
{file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"},
]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "asgiref"
version = "3.8.1"
@ -590,29 +574,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "asttokens"
version = "2.4.1"
description = "Annotate AST trees with source code positions"
optional = false
python-versions = "*"
files = [
{file = "asttokens-2.4.1-py2.py3-none-any.whl", hash = "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24"},
{file = "asttokens-2.4.1.tar.gz", hash = "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"},
]
[package.dependencies]
six = ">=1.12.0"
[package.extras]
astroid = ["astroid (>=1,<2)", "astroid (>=2,<4)"]
test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "async-timeout"
version = "4.0.3"
@ -879,22 +840,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "backcall"
version = "0.2.0"
description = "Specifications for callback functions passed in to an API"
optional = false
python-versions = "*"
files = [
{file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"},
{file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"},
]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "bce-python-sdk"
version = "0.8.87"
@ -2549,25 +2494,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "executing"
version = "2.0.1"
description = "Get the currently executing AST node of a frame, and other information"
optional = false
python-versions = ">=3.5"
files = [
{file = "executing-2.0.1-py2.py3-none-any.whl", hash = "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"},
{file = "executing-2.0.1.tar.gz", hash = "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147"},
]
[package.extras]
tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "fido2"
version = "1.1.3"
@ -3349,49 +3275,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "ipython"
version = "8.14.0"
description = "IPython: Productive Interactive Computing"
optional = false
python-versions = ">=3.9"
files = [
{file = "ipython-8.14.0-py3-none-any.whl", hash = "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"},
{file = "ipython-8.14.0.tar.gz", hash = "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1"},
]
[package.dependencies]
appnope = {version = "*", markers = "sys_platform == \"darwin\""}
backcall = "*"
colorama = {version = "*", markers = "sys_platform == \"win32\""}
decorator = "*"
jedi = ">=0.16"
matplotlib-inline = "*"
pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""}
pickleshare = "*"
prompt-toolkit = ">=3.0.30,<3.0.37 || >3.0.37,<3.1.0"
pygments = ">=2.4.0"
stack-data = "*"
traitlets = ">=5"
[package.extras]
all = ["black", "curio", "docrepr", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.21)", "pandas", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"]
black = ["black"]
doc = ["docrepr", "ipykernel", "matplotlib", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "typing-extensions"]
kernel = ["ipykernel"]
nbconvert = ["nbconvert"]
nbformat = ["nbformat"]
notebook = ["ipywidgets", "notebook"]
parallel = ["ipyparallel"]
qtconsole = ["qtconsole"]
test = ["pytest (<7.1)", "pytest-asyncio", "testpath"]
test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "iso8601"
version = "2.1.0"
@ -3459,30 +3342,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "jedi"
version = "0.19.1"
description = "An autocompletion tool for Python that can be used for text editors."
optional = false
python-versions = ">=3.6"
files = [
{file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"},
{file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"},
]
[package.dependencies]
parso = ">=0.8.3,<0.9.0"
[package.extras]
docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"]
qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "jinja2"
version = "3.1.2"
@ -3972,25 +3831,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "matplotlib-inline"
version = "0.1.7"
description = "Inline Matplotlib backend for Jupyter"
optional = false
python-versions = ">=3.8"
files = [
{file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"},
{file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"},
]
[package.dependencies]
traitlets = "*"
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "maxminddb"
version = "2.6.2"
@ -4768,26 +4608,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "parso"
version = "0.8.4"
description = "A Python Parser"
optional = false
python-versions = ">=3.6"
files = [
{file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"},
{file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"},
]
[package.extras]
qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"]
testing = ["docopt", "pytest"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "passlib"
version = "1.7.4"
@ -4861,22 +4681,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "pickleshare"
version = "0.7.5"
description = "Tiny 'shelve'-like database with concurrency support"
optional = false
python-versions = "*"
files = [
{file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"},
{file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"},
]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "pillow"
version = "10.0.1"
@ -5219,25 +5023,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "pure-eval"
version = "0.2.2"
description = "Safely evaluate AST nodes without side effects"
optional = false
python-versions = "*"
files = [
{file = "pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350"},
{file = "pure_eval-0.2.2.tar.gz", hash = "sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3"},
]
[package.extras]
tests = ["pytest"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "pyasn1"
version = "0.5.0"
@ -6429,27 +6214,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "receptorctl"
version = "1.4.8"
description = "\"Receptorctl is a front-end CLI and importable Python library that interacts with Receptor over its control socket interface.\""
optional = false
python-versions = "*"
files = [
{file = "receptorctl-1.4.8-py3-none-any.whl", hash = "sha256:468d949f85bb50de9fd77cf5af2c173450b346fada787e3907a16f161aff17d6"},
{file = "receptorctl-1.4.8.tar.gz", hash = "sha256:5cf94bec214641506f8f956247c40cb8cc73e84b1a2490a125e49d35159dfa62"},
]
[package.dependencies]
click = "*"
python-dateutil = "*"
pyyaml = "*"
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "redis"
version = "5.0.3"
@ -6968,30 +6732,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "stack-data"
version = "0.6.3"
description = "Extract data from python stack frames and tracebacks for informative displays"
optional = false
python-versions = "*"
files = [
{file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"},
{file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"},
]
[package.dependencies]
asttokens = ">=2.1.0"
executing = ">=1.2.0"
pure-eval = "*"
[package.extras]
tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "stevedore"
version = "5.2.0"
@ -7115,26 +6855,6 @@ type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "traitlets"
version = "5.14.3"
description = "Traitlets Python configuration system"
optional = false
python-versions = ">=3.8"
files = [
{file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"},
{file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"},
]
[package.extras]
docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"]
test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"]
[package.source]
type = "legacy"
url = "https://mirrors.aliyun.com/pypi/simple"
reference = "aliyun"
[[package]]
name = "treelib"
version = "1.6.4"
@ -7981,4 +7701,4 @@ reference = "aliyun"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "54e8ab1d59a70cc6857323145725f13f0ae48eb5f286683bb7d0c7bebb31fb45"
content-hash = "2dc429e66e78ab1e264e26da60df6a67cb8d5e501d80297332d673646bc0cf13"

2
pyproject.toml

@ -135,7 +135,6 @@ pyopenssl = "23.2.0"
redis = { url = "https://github.com/jumpserver-dev/redis-py/archive/refs/tags/v5.0.3.zip" }
pymongo = "4.4.1"
pyfreerdp = "0.0.2"
ipython = "8.14.0"
forgerypy3 = "0.3.1"
django-debug-toolbar = "4.1.0"
pympler = "1.0.1"
@ -156,7 +155,6 @@ xlsxwriter = "^3.1.9"
exchangelib = "^5.1.0"
xmlsec = "^1.3.13"
lxml = "5.2.1"
receptorctl = "^1.4.5"
pydantic = "^2.7.4"
annotated-types = "^0.6.0"
httpx = "^0.27.0"

168
receptor

@ -1,168 +0,0 @@
#!/usr/bin/env python3
# coding: utf-8
import argparse
import logging
import os
import shutil
import signal
import subprocess
import tempfile
from apps.libs.process.ssh import kill_ansible_ssh_process
ANSIBLE_RUNNER_COMMAND = "ansible-runner"
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
APPS_DIR = os.path.join(PROJECT_DIR, 'apps')
TEMP_DIR = os.path.join(PROJECT_DIR, "tmp")
DEFAULT_SHARE_DIR = os.path.join(PROJECT_DIR, "data", "share")
DEFAULT_ANSIBLE_MODULES_DIR = os.path.join(APPS_DIR, "libs", "ansible", "modules")
DEFAULT_CONTROL_SOCK_PATH = os.path.join(DEFAULT_SHARE_DIR, "control.sock")
DEFAULT_TCP_LISTEN_ADDRESS = "0.0.0.0:7521"
logger = logging.getLogger(__name__)
os.chdir(APPS_DIR)
class ReceptorService:
def __init__(self):
self.pid_file = os.path.join(TEMP_DIR, "receptor.pid")
self.receptor_command = [
'receptor',
'--local-only',
'--node', 'id=primary',
'--log-level', 'level=Error',
'--control-service',
'service=control',
'tcplisten={}'.format(DEFAULT_TCP_LISTEN_ADDRESS),
'--work-command',
'worktype={}'.format(ANSIBLE_RUNNER_COMMAND),
'command={}'.format(ANSIBLE_RUNNER_COMMAND),
'params=worker',
'allowruntimeparams=true',
'--work-command',
'worktype={}'.format("kill"),
'command={}'.format("python"),
"params={} kill".format(os.path.join(PROJECT_DIR, "receptor")),
'allowruntimeparams=true'
]
@staticmethod
def before_start():
os.makedirs(os.path.join(DEFAULT_SHARE_DIR), exist_ok=True)
status_dir = os.path.join(tempfile.gettempdir(), "receptor")
if os.path.exists(status_dir):
shutil.rmtree(status_dir)
def start(self):
self.before_start()
if os.path.exists(self.pid_file):
with open(self.pid_file, 'r') as f:
pid_str = f.read()
try:
pid = int(pid_str)
os.kill(pid, 0)
print("\n- Receptor service is already running.")
return
except ProcessLookupError:
print("\n- PID file exists but process does not, starting Receptor...")
except ValueError:
print("\n- PID file is corrupted, starting Receptor...")
os.remove(self.pid_file)
os.environ.setdefault('ANSIBLE_LIBRARY', DEFAULT_ANSIBLE_MODULES_DIR)
os.environ.update({'PYTHONPATH': APPS_DIR})
process = subprocess.Popen(self.receptor_command)
with open(self.pid_file, 'w') as f:
f.write(str(process.pid))
print("\n- Receptor service started successfully.")
def exit_handler(signum, frame):
process.terminate()
process.kill()
signal.signal(signal.SIGINT, exit_handler)
signal.signal(signal.SIGTERM, exit_handler)
process.wait()
def stop(self):
if not os.path.exists(self.pid_file):
print("\n- Receptor service is not running.")
return
with open(self.pid_file, 'r') as f:
pid = int(f.read())
try:
os.kill(pid, signal.SIGTERM)
os.remove(self.pid_file)
print("\n- Receptor service stopped successfully.")
except ProcessLookupError:
print("\n- Failed to stop Receptor service: Process does not exist.")
os.remove(self.pid_file)
def restart(self):
self.stop()
self.start()
def status(self):
if os.path.exists(self.pid_file):
with open(self.pid_file, 'r') as f:
pid_str = f.read()
try:
pid = int(pid_str)
os.kill(pid, 0)
print("\n- Receptor service is running.")
return
except ProcessLookupError:
print("\n- Receptor service is not running.")
else:
print("\n- Receptor service is not running.")
def handle_receptor_action(args):
action = args.action
srv = ReceptorService()
if action == "start":
srv.start()
elif action == 'stop':
srv.stop()
elif action == "restart":
srv.restart()
elif action == "status":
srv.status()
elif action == "kill":
kill_progress_tree()
def kill_progress_tree(pid=None):
if not pid:
try:
pid_input = input()
pid = int(pid_input)
logger.info("progress {} will be kill".format(pid))
kill_ansible_ssh_process(pid)
except Exception as e:
logger.error(e)
return
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="""
Jumpserver receptor service control tools;
"""
)
parser.add_argument(
'action', type=str,
choices=("start", "stop", "restart", "status", "kill"),
help="Action to run"
)
# parser.add_argument('--pid', type=int, default=42, help='what PID you want to kill')
args = parser.parse_args()
handle_receptor_action(args)

36
utils/clean_site_packages.sh

@ -0,0 +1,36 @@
#!/bin/bash
#
lib_path="/opt/py3/lib/python3.11/site-packages"
# 清理不需要的模块
need_clean="jedi botocore/data"
for i in $need_clean; do
rm -rf "${lib_path}/${i}"
done
# 清理 ansible connection 中 不需要的模块
ansible_connection="${lib_path}/ansible_collections"
need_clean="fortinet dellemc f5networks netapp theforeman google azure cyberark ibm
netbox purestorage inspur netapp_eseries sensu check_point vyos arista"
for i in $need_clean; do
echo "rm -rf ${ansible_connection:-tmp}/${i}"
rm -rf "${ansible_connection:-tmp}/${i}"
done
# 清理缓存文件
cd lib_path
find . -name "*.pyc" -exec rm -f {} \;
# 清理不需要的国际化文件
find . -name 'locale' -o -name 'locales' -type d | while read -r dir; do
find "$dir" -mindepth 1 -maxdepth 1 -type d \
! -name 'zh_Hans' \
! -name 'zh_Hant' \
! -name 'zh_CN' \
! -name 'en' \
! -name 'en_US' \
! -name 'ja' \
! -name 'fr' \
-exec rm -rf {} \;
done
Loading…
Cancel
Save