Browse Source

perf: 优化 delay run

pull/9530/head
ibuler 2 years ago
parent
commit
223b73c5c6
  1. 2
      apps/assets/signal_handlers/node_assets_amount.py
  2. 2
      apps/assets/signal_handlers/node_assets_mapping.py
  3. 16
      apps/common/decorators.py
  4. 2
      utils/generate_fake_data/generate.py
  5. 3
      utils/generate_fake_data/resources/base.py

2
apps/assets/signal_handlers/node_assets_amount.py

@ -24,7 +24,6 @@ def on_node_asset_change(sender, action, instance, reverse, pk_set, **kwargs):
if action in refused:
raise ValueError
logger.debug('Recv asset nodes change signal, recompute node assets amount')
mapper = {PRE_ADD: add, POST_REMOVE: sub}
if action not in mapper:
return
@ -40,6 +39,7 @@ def on_node_asset_change(sender, action, instance, reverse, pk_set, **kwargs):
@merge_delay_run(ttl=5)
def update_nodes_assets_amount(node_ids=()):
nodes = list(Node.objects.filter(id__in=node_ids))
logger.debug('Recv asset nodes change signal, recompute node assets amount')
logger.info('Update nodes assets amount: {} nodes'.format(len(node_ids)))
if len(node_ids) > 100:

2
apps/assets/signal_handlers/node_assets_mapping.py

@ -23,6 +23,7 @@ node_assets_mapping_pub_sub = lazy(lambda: RedisPubSub('fm.node_asset_mapping'),
@merge_delay_run(ttl=5)
def expire_node_assets_mapping(org_ids=()):
logger.debug("Recv asset nodes changed signal, expire memery node asset mapping")
# 所有进程清除(自己的 memory 数据)
root_org_id = Organization.ROOT_ID
Node.expire_node_all_asset_ids_cache_mapping(root_org_id)
@ -53,7 +54,6 @@ def on_node_post_delete(sender, instance, **kwargs):
@receiver(m2m_changed, sender=Asset.nodes.through)
def on_node_asset_change(sender, instance, **kwargs):
logger.debug("Recv asset nodes changed signal, expire memery node asset mapping")
expire_node_assets_mapping(org_ids=(instance.org_id,))

16
apps/common/decorators.py

@ -175,13 +175,17 @@ def merge_delay_run(ttl=5, key=None):
key_suffix = suffix_key_func(*args, **kwargs)
cache_key = f'MERGE_DELAY_RUN_{func_name}_{key_suffix}'
cache_kwargs = _loop_debouncer_func_args_cache.get(cache_key, {})
for k, v in kwargs.items():
if not isinstance(v, tuple):
raise ValueError('func kwargs value must be list or tuple: %s' % func.__name__)
if not isinstance(v, (tuple, list, set)):
raise ValueError('func kwargs value must be list or tuple: %s %s' % (func.__name__, v))
if k not in cache_kwargs:
cache_kwargs[k] = v
elif isinstance(v, set):
cache_kwargs[k] = cache_kwargs[k].union(v)
else:
cache_kwargs[k] += v
_loop_debouncer_func_args_cache[cache_key] = cache_kwargs
run_debouncer_func(cache_key, org, ttl, func, *args, **cache_kwargs)
return wrapper
@ -190,11 +194,11 @@ def merge_delay_run(ttl=5, key=None):
@delay_run(ttl=5)
def test_delay_run(*username):
def test_delay_run(username):
print("Hello, %s, now is %s" % (username, time.time()))
@merge_delay_run(ttl=5, key=lambda *users: users[0][0])
@merge_delay_run(ttl=5, key=lambda users=(): users[0][0])
def test_merge_delay_run(users=()):
name = ','.join(users)
time.sleep(2)
@ -206,8 +210,8 @@ def do_test():
print("start : %s" % time.time())
for i in range(100):
# test_delay_run('test', year=i)
test_merge_delay_run('test %s' % i)
test_merge_delay_run('best %s' % i)
test_merge_delay_run(users=['test %s' % i])
test_merge_delay_run(users=['best %s' % i])
test_delay_run('test run %s' % i)
end = time.time()

2
utils/generate_fake_data/generate.py

@ -40,7 +40,7 @@ def main():
default='all',
help="resource to generate"
)
parser.add_argument('-c', '--count', type=int, default=10000)
parser.add_argument('-c', '--count', type=int, default=1000)
parser.add_argument('-b', '--batch_size', type=int, default=100)
parser.add_argument('-o', '--org', type=str, default='')
args = parser.parse_args()

3
utils/generate_fake_data/resources/base.py

@ -44,5 +44,6 @@ class FakeDataGenerator:
using = end - start
from_size = created
created += len(batch)
print('Generate %s: %s-%s [{}s]' % (self.resource, from_size, created, using))
print('Generate %s: %s-%s [%s]' % (self.resource, from_size, created, using))
self.after_generate()
time.sleep(20)

Loading…
Cancel
Save