From d5c5bc219e22e0878a14208bc963b84d969e61f4 Mon Sep 17 00:00:00 2001
From: Boyuan Yao <70263930+Cypher30@users.noreply.github.com>
Date: Fri, 11 Nov 2022 23:17:25 +0800
Subject: [PATCH] [SC] add GPT example for auto checkpoint (#1889)
* [sc] SC tutorial for auto checkpoint
* [sc] polish examples
* [sc] polish readme
* [sc] polish readme and help information
* [sc] polish readme and help information
---
colossalai/fx/passes/meta_info_prop.py | 2 +-
examples/tutorial/auto_parallel/README.md | 79 ++
.../auto_parallel/auto_ckpt_demo.ipynb | 878 ------------------
.../tutorial/auto_parallel/bench_utils.py | 111 ++-
.../auto_parallel/demo_gpt2_medium.py | 108 +++
.../tutorial/auto_parallel/demo_resnet152.py | 74 ++
.../tutorial/auto_parallel/demo_resnet50.py | 107 +++
.../auto_parallel/imgs/gpt2_benchmark.png | Bin 0 -> 66851 bytes
.../auto_parallel/imgs/resnet50_benchmark.png | Bin 0 -> 72546 bytes
9 files changed, 470 insertions(+), 889 deletions(-)
delete mode 100644 examples/tutorial/auto_parallel/auto_ckpt_demo.ipynb
create mode 100644 examples/tutorial/auto_parallel/demo_gpt2_medium.py
create mode 100644 examples/tutorial/auto_parallel/demo_resnet152.py
create mode 100644 examples/tutorial/auto_parallel/demo_resnet50.py
create mode 100644 examples/tutorial/auto_parallel/imgs/gpt2_benchmark.png
create mode 100644 examples/tutorial/auto_parallel/imgs/resnet50_benchmark.png
diff --git a/colossalai/fx/passes/meta_info_prop.py b/colossalai/fx/passes/meta_info_prop.py
index 711439955..5137494ad 100644
--- a/colossalai/fx/passes/meta_info_prop.py
+++ b/colossalai/fx/passes/meta_info_prop.py
@@ -338,7 +338,7 @@ def metainfo_trace(gm: torch.fx.GraphModule, *args, verbose: bool = False, unit:
Returns:
torch.fx.GraphModule: The ``GraphModule`` annotated with MetaInfo.
"""
- device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+ device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
interp = MetaInfoProp(gm.to(device))
if is_compatible_with_meta():
from colossalai.fx.profiler import MetaTensor
diff --git a/examples/tutorial/auto_parallel/README.md b/examples/tutorial/auto_parallel/README.md
index bed488022..e93a8288b 100644
--- a/examples/tutorial/auto_parallel/README.md
+++ b/examples/tutorial/auto_parallel/README.md
@@ -15,3 +15,82 @@ export DATA=/path/to/data
```bash
colossalai run --nproc_per_node 4 auto_parallel_demo.py
```
+
+## Auto Checkpoint Benchmarking
+
+We prepare three demos for you to test the performance of auto checkpoint, the test `demo_resnet50.py` and `demo_gpt2_medium.py` will show you the ability of solver to search checkpoint strategy that could fit in the given budget.
+
+The usage of the above two test
+```bash
+python demo_resnet50.py --help
+usage: ResNet50 Auto Activation Benchmark [-h] [--batch_size BATCH_SIZE] [--num_steps NUM_STEPS] [--sample_points SAMPLE_POINTS] [--free_memory FREE_MEMORY]
+ [--start_factor START_FACTOR]
+
+optional arguments:
+ -h, --help show this help message and exit
+ --batch_size BATCH_SIZE
+ batch size for benchmark, default 128
+ --num_steps NUM_STEPS
+ number of test steps for benchmark, default 5
+ --sample_points SAMPLE_POINTS
+ number of sample points for benchmark from start memory budget to maximum memory budget (free_memory), default 15
+ --free_memory FREE_MEMORY
+ maximum memory budget in MB for benchmark, default 11000 MB
+ --start_factor START_FACTOR
+ start memory budget factor for benchmark, the start memory budget will be free_memory / start_factor, default 4
+
+# run with default settings
+python demo_resnet50.py
+
+python demo_gpt2_medium.py --help
+usage: GPT2 medium Auto Activation Benchmark [-h] [--batch_size BATCH_SIZE] [--num_steps NUM_STEPS] [--sample_points SAMPLE_POINTS] [--free_memory FREE_MEMORY]
+ [--start_factor START_FACTOR]
+
+optional arguments:
+ -h, --help show this help message and exit
+ --batch_size BATCH_SIZE
+ batch size for benchmark, default 8
+ --num_steps NUM_STEPS
+ number of test steps for benchmark, default 5
+ --sample_points SAMPLE_POINTS
+ number of sample points for benchmark from start memory budget to maximum memory budget (free_memory), default 15
+ --free_memory FREE_MEMORY
+ maximum memory budget in MB for benchmark, default 56000 MB
+ --start_factor START_FACTOR
+ start memory budget factor for benchmark, the start memory budget will be free_memory / start_factor, default 10
+
+# run with default settings
+python demo_gpt2_medium.py
+```
+
+There are some results for your reference
+
+### ResNet 50
+![](./imgs/resnet50_benchmark.png)
+
+### GPT2 Medium
+![](./imgs/gpt2_benchmark.png)
+
+We also prepare the demo `demo_resnet152.py` to manifest the benefit of auto activation with large batch, the usage is listed as follows
+```bash
+python demo_resnet152.py --help
+usage: ResNet152 Auto Activation Through Put Benchmark [-h] [--num_steps NUM_STEPS]
+
+optional arguments:
+ -h, --help show this help message and exit
+ --num_steps NUM_STEPS
+ number of test steps for benchmark, default 5
+
+# run with default settings
+python demo_resnet152.py
+```
+
+here are some results on our end for your reference
+```bash
+===============test summary================
+batch_size: 512, peak memory: 73314.392 MB, through put: 254.286 images/s
+batch_size: 1024, peak memory: 73316.216 MB, through put: 397.608 images/s
+batch_size: 2048, peak memory: 72927.837 MB, through put: 277.429 images/s
+```
+
+The above tests will output the test summary and a plot of the benchmarking results.
diff --git a/examples/tutorial/auto_parallel/auto_ckpt_demo.ipynb b/examples/tutorial/auto_parallel/auto_ckpt_demo.ipynb
deleted file mode 100644
index cacf5d5f3..000000000
--- a/examples/tutorial/auto_parallel/auto_ckpt_demo.ipynb
+++ /dev/null
@@ -1,878 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/home/lcsjy/.conda/envs/autoparallel/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
- " from .autonotebook import tqdm as notebook_tqdm\n"
- ]
- },
- {
- "data": {
- "text/html": [
- "
[11/10/22 18:04:14] INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:1 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m[11/10/22 18:04:14]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m1\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:1 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m1\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:2 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m2\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:2 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m2\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:3 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m3\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:3 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m3\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:4 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m4\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:4 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m4\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:5 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m5\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:5 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m5\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:6 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m6\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:6 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m6\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:7 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m7\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:7 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m7\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- " store_based_barrier_key:8 to store for rank: 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Added key: \n",
- "\u001b[2;36m \u001b[0m store_based_barrier_key:\u001b[1;36m8\u001b[0m to store for rank: \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - torch.distributed.distributed_c10d - INFO: Rank 0 : Completed store-based \n",
- " barrier for key:store_based_barrier_key:8 with 1 nodes. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - torch.distributed.distributed_c10d - INFO: Rank \u001b[1;36m0\u001b[0m: Completed store-based \n",
- "\u001b[2;36m \u001b[0m barrier for key:store_based_barrier_key:\u001b[1;36m8\u001b[0m with \u001b[1;36m1\u001b[0m nodes. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: \n",
- " /home/lcsjy/ColossalAI/colossalai/context/ parallel_context.py :521 set_device \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/context/\u001b[0m\u001b[95mparallel_context.py\u001b[0m:\u001b[1;36m521\u001b[0m set_device \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: process rank 0 is bound to device 0 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: process rank \u001b[1;36m0\u001b[0m is bound to device \u001b[1;36m0\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: \n",
- " /home/lcsjy/ColossalAI/colossalai/context/ parallel_context.py :557 set_seed \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/context/\u001b[0m\u001b[95mparallel_context.py\u001b[0m:\u001b[1;36m557\u001b[0m set_seed \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: initialized seed on rank 0 , numpy: 1024 , python \n",
- " random: 1024 , ParallelMode.DATA: 1024 , ParallelMode.TENSOR: 1024 ,the default parallel \n",
- " seed is ParallelMode.DATA. \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: initialized seed on rank \u001b[1;36m0\u001b[0m, numpy: \u001b[1;36m1024\u001b[0m, python \n",
- "\u001b[2;36m \u001b[0m random: \u001b[1;36m1024\u001b[0m, ParallelMode.DATA: \u001b[1;36m1024\u001b[0m, ParallelMode.TENSOR: \u001b[1;36m1024\u001b[0m,the default parallel \n",
- "\u001b[2;36m \u001b[0m seed is ParallelMode.DATA. \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: /home/lcsjy/ColossalAI/colossalai/ initialize.py :117 \n",
- " launch \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: \u001b[35m/home/lcsjy/ColossalAI/colossalai/\u001b[0m\u001b[95minitialize.py\u001b[0m:\u001b[1;36m117\u001b[0m \n",
- "\u001b[2;36m \u001b[0m launch \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " INFO colossalai - colossalai - INFO: Distributed environment is initialized, data parallel \n",
- " size: 1 , pipeline parallel size: 1 , tensor parallel size: 1 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m colossalai - colossalai - INFO: Distributed environment is initialized, data parallel \n",
- "\u001b[2;36m \u001b[0m size: \u001b[1;36m1\u001b[0m, pipeline parallel size: \u001b[1;36m1\u001b[0m, tensor parallel size: \u001b[1;36m1\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "import time\n",
- "import torchvision.models as tm\n",
- "import torch\n",
- "import colossalai\n",
- "from colossalai.fx import symbolic_trace, metainfo_trace\n",
- "from colossalai.auto_parallel.checkpoint import CheckpointSolverRotor\n",
- "from functools import partial\n",
- "from colossalai.utils import free_port\n",
- "\n",
- "from bench_utils import bench, bench_rotor\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "colossalai.launch(config={}, rank=0, world_size=1, host='localhost', port=free_port(), backend='nccl')"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### ResNet152 with batch size = 512 fails"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(78990.4404296875, inf)"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "def data_gen(batch_size, shape, device='cuda'):\n",
- " data = torch.empty(batch_size, *shape, device=device)\n",
- " label = torch.empty(batch_size, dtype=torch.long, device=device).random_(1000)\n",
- " return {'x': data}, label\n",
- "\n",
- "model = tm.resnet152()\n",
- "gm = symbolic_trace(model)\n",
- "gm = metainfo_trace(gm, torch.empty(512, 3, 224, 224, device='meta'))\n",
- "bench(gm, torch.nn.CrossEntropyLoss(), partial(data_gen, batch_size=512, shape=(3, 224, 224)), num_steps=5)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### ResNet152 with batch size = 2048 succeeds "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(74495.8486328125, 5634.262561798096)"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "def data_gen(batch_size, shape, device='cuda'):\n",
- " data = torch.empty(batch_size, *shape, device=device)\n",
- " label = torch.empty(batch_size, dtype=torch.long, device=device).random_(1000)\n",
- " return {'x': data}, label\n",
- "\n",
- "model = tm.resnet152()\n",
- "gm = symbolic_trace(model)\n",
- "gm = metainfo_trace(gm, torch.empty(2048, 3, 224, 224, device='meta'))\n",
- "solver = CheckpointSolverRotor(gm.graph, free_memory=torch.cuda.mem_get_info(device=0)[0] * 0.95)\n",
- "gm.graph = solver.solve()\n",
- "bench(gm, torch.nn.CrossEntropyLoss(), partial(data_gen, batch_size=2048, shape=(3, 224, 224)), num_steps=5)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Benchmarking on ResNet18"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "[11/10/22 18:04:20] WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m[11/10/22 18:04:20]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- "[11/10/22 18:04:21] WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m[11/10/22 18:04:21]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- "[11/10/22 18:04:22] WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m[11/10/22 18:04:22]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- "[11/10/22 18:04:23] WARNING colossalai - colossalai - WARNING: \n",
- " /home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/ ckpt_solver_rotor.py :82 \n",
- " solve \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m[11/10/22 18:04:23]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: \n",
- "\u001b[2;36m \u001b[0m \u001b[35m/home/lcsjy/ColossalAI/colossalai/auto_parallel/checkpoint/\u001b[0m\u001b[95mckpt_solver_rotor.py\u001b[0m:\u001b[1;36m82\u001b[0m \n",
- "\u001b[2;36m \u001b[0m solve \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "text/html": [
- " WARNING colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- " chain from index 0 to 14 with memory 500 \n",
- " \n"
- ],
- "text/plain": [
- "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m colossalai - colossalai - WARNING: Checkpoint solver failed: Can not process this \n",
- "\u001b[2;36m \u001b[0m chain from index \u001b[1;36m0\u001b[0m to \u001b[1;36m14\u001b[0m with memory \u001b[1;36m500\u001b[0m \n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "def data_gen(batch_size, shape, device='cuda'):\n",
- " data = torch.empty(batch_size, *shape, device=device)\n",
- " label = torch.empty(batch_size, dtype=torch.long, device=device).random_(1000)\n",
- " return (data, ), label\n",
- "\n",
- "model = tm.resnet18()\n",
- "gm = symbolic_trace(model)\n",
- "gm = metainfo_trace(gm, torch.empty(128, 3, 224, 224, device='meta'))\n",
- "peak_hist, step_hist = bench_rotor(gm, torch.nn.CrossEntropyLoss(), partial(data_gen, batch_size=128, shape=(3, 224, 224)), num_steps=5, sample_points=20, free_memory=2700 * 1024**2)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[]"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAArEAAAKTCAYAAAAOvlAQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvJElEQVR4nO3df5BV9X34/9eVZRfF5TaKK4Ir/kjQRUwQZSBkI0EjP0TBZYaiYxQCsbEBCST1E62xHW3aJTTNdGIaCXHdsaEqo+JWhRjZjiAO2hJQEycUcKMsVQzByK4UC9E93z8y3m+uu/zYVYQ3PB4zZ+bes+/z4+57Dnl6e/Y0l2VZFgAAkJBjDvUJAABAZ4lYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEhOyaE+gY9TW1tbvP7661FeXh65XO5Qnw4AAB+QZVm8/fbb0bdv3zjmmL1/33pURezrr78elZWVh/o0AADYjy1btsSpp566158fVRFbXl4eEX/8pfTq1esQnw0AAB/U2toalZWVhW7bm6MqYt+/haBXr14iFgDgMLa/Wz/9YRcAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELkLC2tiza2rJDfRoAHzsRC5CotrYszvzrZXHmXy8TssBRR8QCJOr3u/Z0+BrgaCBiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAktOpiD399NMjl8u1W2bOnFkYs379+pgwYULk8/koLy+P4cOHR3Nzc+Hnb7zxRlx77bXRp0+f6NmzZwwZMiQeeuih/R77Rz/6UZxxxhnRo0ePuOCCC2LVqlWdOXUAAI4gnYrYNWvWxNatWwvL8uXLIyJi8uTJERHR1NQU1dXVcc4558SKFSvixRdfjNtuuy169OhR2Me1114bGzZsiEcffTR+9atfxaRJk2LKlCnx/PPP7/W4ixcvjjlz5sStt94azz//fHz+85+PcePGFcUxAABHj1yWZVlXN54zZ048/vjjsWnTpsjlcnHVVVdF9+7d46c//eletzn++OPjrrvuimuvvbaw7sQTT4z58+fHjBkzOtxm2LBhMWTIkLjrrrsK66qqquLKK6+M2travR5r9+7dsXv37sL71tbWqKysjJaWlujVq1dnPirAYWf7zt1x4XcaIyLiF9/+YvQ+vuwQnxHAh9fa2hr5fH6/vdble2L37NkTixYtiunTp0cul4u2trZYunRpDBgwIMaMGRMVFRUxbNiwaGhoKNquuro6Fi9eHL///e+jra0tHnjggdi9e3d84Qtf2Otx1q5dG6NHjy5aP3r06Fi9evU+z7G2tjby+Xxhqays7OrHBQDgMNLliG1oaIgdO3bEtGnTIiJi27ZtsXPnzpg3b16MHTs2nnzyyaipqYlJkybFypUrC9stXrw43n333TjxxBOjrKwsvvrVr8YjjzwSZ511VofH2b59e7z33ntx8sknF60/+eST44033tjnOd5yyy3R0tJSWLZs2dLVjwsAwGGkpKsb1tXVxbhx46Jv374REdHW1hYRERMnToy5c+dGRMTgwYNj9erVsWDBghg5cmRERHz729+Ot956KxobG6N3797R0NAQkydPjlWrVsV555231+Plcrmi91mWtVv3QWVlZVFW5v+8BgBwpOlSxG7evDkaGxtjyZIlhXW9e/eOkpKSGDhwYNHYqqqqeOaZZyLij3/49cMf/jBeeumlOPfccyMi4jOf+UysWrUq/uVf/iUWLFjQ7li9e/eObt26tfvWddu2be2+nQUA4OjQpdsJ6uvro6KiIsaPH19YV1paGkOHDo0NGzYUjd24cWP0798/IiJ27dr1x4MeU3zYbt26Fb7J/aDS0tK44IILCk9CeN/y5ctjxIgRXTl9AAAS1+lvYtva2qK+vj6mTp0aJSXFm990000xZcqUuOiii2LUqFHxxBNPxGOPPRYrVqyIiIhzzjknPvnJT8ZXv/rV+N73vhcnnnhiNDQ0xPLly+Pxxx8v7OeSSy6JmpqamDVrVkREfOMb34hrr702LrzwwvjsZz8bCxcujObm5rjhhhs+xEcHACBVnY7YxsbGaG5ujunTp7f7WU1NTSxYsCBqa2tj9uzZcfbZZ8fDDz8c1dXVERHRvXv3WLZsWdx8881xxRVXxM6dO+OTn/xk3HvvvXHZZZcV9tPU1BTbt28vvJ8yZUq8+eabcccdd8TWrVtj0KBBsWzZssI3vAAAHF0+1HNiU3Ogzx0DSIHnxAJHooP+nFgAADhURCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAiTq2O7dOnwNcDQQsQCJyuU6fg1wNBCxAAAkR8QCAJAcEQsAQHJELAAAyRGxAAAkR8QCAJAcEQsAQHJELAAAyRGxAAAkR8QCAJAcEQsAQHJELECi2rKOXwMcDUQsQKLe+t89Hb4GOBqIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWIBEtWVZh68BjgYiFiBRv//fPR2+BjgaiFgAAJIjYgEASE6nIvb000+PXC7Xbpk5c2ZhzPr162PChAmRz+ejvLw8hg8fHs3NzRER8eqrr3a4fS6XiwcffHCvx3333Xfj29/+dpxxxhlx7LHHxplnnhl33HFHtLW1dfFjAwCQspLODF6zZk289957hfcvvfRSXHrppTF58uSIiGhqaorq6uqYMWNG3H777ZHP52P9+vXRo0ePiIiorKyMrVu3Fu1z4cKFMX/+/Bg3btxej/vd7343FixYEPfee2+ce+658Ytf/CK+/OUvRz6fj69//eud+QgAABwBOhWxJ510UtH7efPmxVlnnRUjR46MiIhbb701Lrvsspg/f35hzJlnnll43a1bt+jTp0/RPh555JGYMmVKHH/88Xs97rPPPhsTJ06M8ePHR8QfvxG+//774xe/+EVnTh8AgCNEl++J3bNnTyxatCimT58euVwu2traYunSpTFgwIAYM2ZMVFRUxLBhw6KhoWGv+1i7dm288MILMWPGjH0eq7q6Ov7jP/4jNm7cGBERL774YjzzzDNx2WWX7XO73bt3R2tra9ECAED6uhyxDQ0NsWPHjpg2bVpERGzbti127twZ8+bNi7Fjx8aTTz4ZNTU1MWnSpFi5cmWH+6irq4uqqqoYMWLEPo/1rW99K66++uo455xzonv37nH++efHnDlz4uqrr97ndrW1tZHP5wtLZWVllz4rAACHly5HbF1dXYwbNy769u0bEVH4I6uJEyfG3LlzY/DgwXHzzTfH5ZdfHgsWLGi3/TvvvBP33Xfffr+FjYhYvHhxLFq0KO67775Yt25d3HvvvfG9730v7r333n1ud8stt0RLS0th2bJlSxc+KQAAh5tO3RP7vs2bN0djY2MsWbKksK53795RUlISAwcOLBpbVVUVzzzzTLt9PPTQQ7Fr16647rrr9nu8m266KW6++ea46qqrIiLivPPOi82bN0dtbW1MnTp1r9uVlZVFWVnZgX4sAAAS0aVvYuvr66OioqLwh1YREaWlpTF06NDYsGFD0diNGzdG//792+2jrq4uJkyY0O6PxTqya9euOOaY4lPt1q2bR2wBR7UTepZ2+BrgaNDpb2Lb2tqivr4+pk6dGiUlxZvfdNNNMWXKlLjoooti1KhR8cQTT8Rjjz0WK1asKBr38ssvx9NPPx3Lli3r8BiXXHJJ1NTUxKxZsyIi4oorroi///u/j9NOOy3OPffceP755+P73/9+TJ8+vbOnD3DEOCaX6/A1wNGg0xHb2NgYzc3NHQZkTU1NLFiwIGpra2P27Nlx9tlnx8MPPxzV1dVF4+65557o169fjB49usNjNDU1xfbt2wvv77zzzrjtttvia1/7Wmzbti369u0bX/3qV+Nv/uZvOnv6AAAcAXJZlmWH+iQ+Lq2trZHP56OlpSV69ep1qE8H4EPZ8vtd8fn5T0VExKr/NyoqTzjuEJ8RwId3oL3W5acTAADAoSJiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5nYrY008/PXK5XLtl5syZhTHr16+PCRMmRD6fj/Ly8hg+fHg0NzdHRMSrr77a4fa5XC4efPDBfR77tddeiy996Utx4oknxnHHHReDBw+OtWvXduEjAwCQupLODF6zZk289957hfcvvfRSXHrppTF58uSIiGhqaorq6uqYMWNG3H777ZHP52P9+vXRo0ePiIiorKyMrVu3Fu1z4cKFMX/+/Bg3btxej/vWW2/F5z73uRg1alT87Gc/i4qKimhqaoo/+7M/68zpAwBwhOhUxJ500klF7+fNmxdnnXVWjBw5MiIibr311rjsssti/vz5hTFnnnlm4XW3bt2iT58+Rft45JFHYsqUKXH88cfv9bjf/e53o7KyMurr6wvrTj/99M6cOgAAR5Au3xO7Z8+eWLRoUUyfPj1yuVy0tbXF0qVLY8CAATFmzJioqKiIYcOGRUNDw173sXbt2njhhRdixowZ+zzWo48+GhdeeGFMnjw5Kioq4vzzz4+f/OQn+z3H3bt3R2tra9ECAED6uhyxDQ0NsWPHjpg2bVpERGzbti127twZ8+bNi7Fjx8aTTz4ZNTU1MWnSpFi5cmWH+6irq4uqqqoYMWLEPo/1m9/8Ju6666741Kc+FT//+c/jhhtuiNmzZ8e//uu/7nO72trayOfzhaWysrJLnxUAgMNLLsuyrCsbjhkzJkpLS+Oxxx6LiIjXX389+vXrF1dffXXcd999hXETJkyInj17xv3331+0/TvvvBOnnHJK3HbbbfHNb35zn8cqLS2NCy+8MFavXl1YN3v27FizZk08++yze91u9+7dsXv37sL71tbWqKysjJaWlujVq1enPi/A4WbL73fF5+c/FRERq/7fqKg84bhDfEYAH15ra2vk8/n99lqXvondvHlzNDY2xle+8pXCut69e0dJSUkMHDiwaGxVVVXh6QR/6qGHHopdu3bFddddt9/jnXLKKQe83z9VVlYWvXr1KloAAEhflyK2vr4+KioqYvz48YV1paWlMXTo0NiwYUPR2I0bN0b//v3b7aOuri4mTJjQ7o/FOvK5z33ugPcLAMCRr1NPJ4iIaGtri/r6+pg6dWqUlBRvftNNN8WUKVPioosuilGjRsUTTzwRjz32WKxYsaJo3MsvvxxPP/10LFu2rMNjXHLJJVFTUxOzZs2KiIi5c+fGiBEj4h/+4R/iz//8z+O//uu/YuHChbFw4cLOnj4AAEeATn8T29jYGM3NzTF9+vR2P6upqYkFCxbE/Pnz47zzzou77747Hn744aiuri4ad88990S/fv1i9OjRHR6jqakptm/fXng/dOjQeOSRR+L++++PQYMGxd/93d/FP//zP8c111zT2dMHAOAI0OU/7ErRgd4oDJACf9gFHIkO6h92AQDAoSRiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYAgOSIWAAAkiNiAQBIjogFACA5IhYgUZ/oWdrha4CjgYgFSNQxuY5fAxwNRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMkRsQAAJEfEAgCQHBELAEByRCwAAMnpVMSefvrpkcvl2i0zZ84sjFm/fn1MmDAh8vl8lJeXx/Dhw6O5uTkiIl599dUOt8/lcvHggw8e0DnU1tZGLpeLOXPmdObUAQA4gpR0ZvCaNWvivffeK7x/6aWX4tJLL43JkydHRERTU1NUV1fHjBkz4vbbb498Ph/r16+PHj16REREZWVlbN26tWifCxcujPnz58e4ceMO6PgLFy6MT3/60505bQAAjjCditiTTjqp6P28efPirLPOipEjR0ZExK233hqXXXZZzJ8/vzDmzDPPLLzu1q1b9OnTp2gfjzzySEyZMiWOP/74fR57586dcc0118RPfvKT+M53vtOZ0wYA4AjT5Xti9+zZE4sWLYrp06dHLpeLtra2WLp0aQwYMCDGjBkTFRUVMWzYsGhoaNjrPtauXRsvvPBCzJgxY7/HmzlzZowfPz6++MUvHvA57t69O1pbW4sWAADS1+WIbWhoiB07dsS0adMiImLbtm2xc+fOmDdvXowdOzaefPLJqKmpiUmTJsXKlSs73EddXV1UVVXFiBEj9nmsBx54INatWxe1tbWdOsfa2trI5/OFpbKyslPbAwBweOpyxNbV1cW4ceOib9++ERHR1tYWERETJ06MuXPnxuDBg+Pmm2+Oyy+/PBYsWNBu+3feeSfuu+++/X4Lu2XLlvj6178eixYtKtxbe6BuueWWaGlpKSxbtmzp1PYAAByeOnVP7Ps2b94cjY2NsWTJksK63r17R0lJSQwcOLBobFVVVTzzzDPt9vHQQw/Frl274rrrrtvnsdauXRvbtm2LCy64oLDuvffei6effjp++MMfxu7du6Nbt24dbltWVhZlZWWd+WgAACSgSxFbX18fFRUVMX78+MK60tLSGDp0aGzYsKFo7MaNG6N///7t9lFXVxcTJkxo98diH3TJJZfEr371q6J1X/7yl+Occ86Jb33rW3sNWAAAjlydjti2traor6+PqVOnRklJ8eY33XRTTJkyJS666KIYNWpUPPHEE/HYY4/FihUrisa9/PLL8fTTT8eyZcs6PMYll1wSNTU1MWvWrCgvL49BgwYV/bxnz55x4okntlsPAMDRodP3xDY2NkZzc3NMnz693c9qampiwYIFMX/+/DjvvPPi7rvvjocffjiqq6uLxt1zzz3Rr1+/GD16dIfHaGpqiu3bt3f21AAAOErksizLDvVJfFxaW1sjn89HS0tL9OrV61CfDsCHsmvPuzHwb34eERG/vmNMHFfapTvEAA4rB9prXX46AQAAHCoiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACS06mIPf300yOXy7VbZs6cWRizfv36mDBhQuTz+SgvL4/hw4dHc3NzRES8+uqrHW6fy+XiwQcf3Otxa2trY+jQoVFeXh4VFRVx5ZVXxoYNG7r4kQEASF2nInbNmjWxdevWwrJ8+fKIiJg8eXJERDQ1NUV1dXWcc845sWLFinjxxRfjtttuix49ekRERGVlZdH2W7dujdtvvz169uwZ48aN2+txV65cGTNnzoznnnsuli9fHu+++26MHj06/vd//7ernxsAgITlsizLurrxnDlz4vHHH49NmzZFLpeLq666Krp37x4//elPD3gf559/fgwZMiTq6uoOeJvf/e53UVFREStXroyLLrrogLdrbW2NfD4fLS0t0atXrwPeDuBwtGvPuzHwb34eERG/vmNMHFdacojPCODDO9Be6/I9sXv27IlFixbF9OnTI5fLRVtbWyxdujQGDBgQY8aMiYqKihg2bFg0NDTsdR9r166NF154IWbMmNGpY7e0tERExAknnLDPcbt3747W1taiBQCA9HU5YhsaGmLHjh0xbdq0iIjYtm1b7Ny5M+bNmxdjx46NJ598MmpqamLSpEmxcuXKDvdRV1cXVVVVMWLEiAM+bpZl8Y1vfCOqq6tj0KBB+xxbW1sb+Xy+sFRWVh7wcQAAOHx1OWLr6upi3Lhx0bdv34iIaGtri4iIiRMnxty5c2Pw4MFx8803x+WXXx4LFixot/0777wT9913X6e/hZ01a1b88pe/jPvvv3+/Y2+55ZZoaWkpLFu2bOnUsQAAODx16QaqzZs3R2NjYyxZsqSwrnfv3lFSUhIDBw4sGltVVRXPPPNMu3089NBDsWvXrrjuuusO+Lg33nhjPProo/H000/Hqaeeut/xZWVlUVZWdsD7BwAgDV2K2Pr6+qioqIjx48cX1pWWlsbQoUPbPfpq48aN0b9//3b7qKuriwkTJsRJJ5203+NlWRY33nhjPPLII7FixYo444wzunLaAAAcITodsW1tbVFfXx9Tp06NkpLizW+66aaYMmVKXHTRRTFq1Kh44okn4rHHHosVK1YUjXv55Zfj6aefjmXLlnV4jEsuuSRqampi1qxZERExc+bMuO++++Lf//3fo7y8PN54442IiMjn83Hsscd29iMAAJC4Tt8T29jYGM3NzTF9+vR2P6upqYkFCxbE/Pnz47zzzou77747Hn744aiuri4ad88990S/fv1i9OjRHR6jqakptm/fXnh/1113RUtLS3zhC1+IU045pbAsXry4s6cPAMAR4EM9JzY1nhMLHEk8JxY4Eh3058QCAMChImIBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDklh/oEAOiaY7t3i1/fMabwGuBoImIBEpXL5eK4Uv+MA0cntxMAAJAcEQsAQHJELAAAyRGxAAAkR8QCAJAcEQsAQHJELAAAyRGxAAAkR8QCAJAcEQsAQHJELAAAyRGxAAAkR8QCAJAcEQsAQHJELAAAyRGxAAAkR8QCAJCcTkXs6aefHrlcrt0yc+bMwpj169fHhAkTIp/PR3l5eQwfPjyam5sjIuLVV1/tcPtcLhcPPvjgPo/9ox/9KM4444zo0aNHXHDBBbFq1aoufFwAAI4EnYrYNWvWxNatWwvL8uXLIyJi8uTJERHR1NQU1dXVcc4558SKFSvixRdfjNtuuy169OgRERGVlZVF22/dujVuv/326NmzZ4wbN26vx128eHHMmTMnbr311nj++efj85//fIwbN64QxwAAHF1yWZZlXd14zpw58fjjj8emTZsil8vFVVddFd27d4+f/vSnB7yP888/P4YMGRJ1dXV7HTNs2LAYMmRI3HXXXYV1VVVVceWVV0Ztbe0BH6u1tTXy+Xy0tLREr169Dng7AAA+Hgfaa12+J3bPnj2xaNGimD59euRyuWhra4ulS5fGgAEDYsyYMVFRURHDhg2LhoaGve5j7dq18cILL8SMGTP2eZy1a9fG6NGji9aPHj06Vq9evc9z3L17d7S2thYtAACkr8sR29DQEDt27Ihp06ZFRMS2bdti586dMW/evBg7dmw8+eSTUVNTE5MmTYqVK1d2uI+6urqoqqqKESNG7PU427dvj/feey9OPvnkovUnn3xyvPHGG/s8x9ra2sjn84WlsrKycx8SAIDDUpcjtq6uLsaNGxd9+/aNiIi2traIiJg4cWLMnTs3Bg8eHDfffHNcfvnlsWDBgnbbv/POO3Hfffft81vYP5XL5YreZ1nWbt0H3XLLLdHS0lJYtmzZckDHAgDg8FbSlY02b94cjY2NsWTJksK63r17R0lJSQwcOLBobFVVVTzzzDPt9vHQQw/Frl274rrrrtvnsXr37h3dunVr963rtm3b2n07+0FlZWVRVla2v48DAEBiuhSx9fX1UVFREePHjy+sKy0tjaFDh8aGDRuKxm7cuDH69+/fbh91dXUxYcKEOOmkk/Z5rNLS0rjgggti+fLlUVNTU1i/fPnymDhxYqfO+/2/YXNvLADA4en9TtvvsweyTnrvvfey0047LfvWt77V7mdLlizJunfvni1cuDDbtGlTduedd2bdunXLVq1aVTRu06ZNWS6Xy372s591eIyLL744u/POOwvvH3jggax79+5ZXV1d9utf/zqbM2dO1rNnz+zVV1/t1Llv2bIliwiLxWKxWCwWy2G+bNmyZZ9d1+lvYhsbG6O5uTmmT5/e7mc1NTWxYMGCqK2tjdmzZ8fZZ58dDz/8cFRXVxeNu+eee6Jfv37tnjjwvqampti+fXvh/ZQpU+LNN9+MO+64I7Zu3RqDBg2KZcuWdfgN77707ds3tmzZEuXl5fu9nzYlra2tUVlZGVu2bPHosMOUOUqDeTr8maM0mKfD3+E8R1mWxdtvv134u6u9+VDPieXw4Pm3hz9zlAbzdPgzR2kwT4e/I2GOuvx0AgAAOFRELAAAyRGxR4CysrL427/9W48TO4yZozSYp8OfOUqDeTr8HQlz5J5YAACS45tYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IPgdra2hg6dGiUl5dHRUVFXHnllbFhw4aiMdOmTYtcLle0DB8+vGjMF77whXZjrrrqqqIxb731Vlx77bWRz+cjn8/HtddeGzt27Cga09zcHFdccUX07NkzevfuHbNnz449e/YclM+eigOZo4iI9evXx4QJEyKfz0d5eXkMHz48mpubCz/fvXt33HjjjdG7d+/o2bNnTJgwIf7nf/6naB/mqOs+qnlyLR08BzJHH/zdv7/84z/+Y2GMa+ng+qjmybV08BzIHO3cuTNmzZoVp556ahx77LFRVVUVd911V9GYI+payvjYjRkzJquvr89eeuml7IUXXsjGjx+fnXbaadnOnTsLY6ZOnZqNHTs227p1a2F58803i/YzcuTI7Prrry8as2PHjqIxY8eOzQYNGpStXr06W716dTZo0KDs8ssvL/z83XffzQYNGpSNGjUqW7duXbZ8+fKsb9++2axZsw7uL+EwdyBz9PLLL2cnnHBCdtNNN2Xr1q3Lmpqasscffzz77W9/Wxhzww03ZP369cuWL1+erVu3Lhs1alT2mc98Jnv33XcLY8xR131U8+RaOngOZI7+9Pe+devW7J577slyuVzW1NRUGONaOrg+qnlyLR08BzJHX/nKV7Kzzjore+qpp7JXXnkl+/GPf5x169Yta2hoKIw5kq4lEXsY2LZtWxYR2cqVKwvrpk6dmk2cOHGf240cOTL7+te/vtef//rXv84iInvuuecK65599tksIrL//u//zrIsy5YtW5Ydc8wx2WuvvVYYc//992dlZWVZS0tL1z7QEaijOZoyZUr2pS99aa/b7NixI+vevXv2wAMPFNa99tpr2THHHJM98cQTWZaZo49aV+Ypy1xLH6eO5uiDJk6cmF188cWF966lj19X5inLXEsfp47m6Nxzz83uuOOOonFDhgzJvv3tb2dZduRdS24nOAy0tLRERMQJJ5xQtH7FihVRUVERAwYMiOuvvz62bdvWbtt/+7d/i969e8e5554bf/VXfxVvv/124WfPPvts5PP5GDZsWGHd8OHDI5/Px+rVqwtjBg0aFH379i2MGTNmTOzevTvWrl37kX7OlH1wjtra2mLp0qUxYMCAGDNmTFRUVMSwYcOioaGhsM3atWvjD3/4Q4wePbqwrm/fvjFo0KCi3785+uh0ZZ7e51r6eOzt37v3/fa3v42lS5fGjBkzCutcSx+/rszT+1xLH4+O5qi6ujoeffTReO211yLLsnjqqadi48aNMWbMmIg48q6lko/tSHQoy7L4xje+EdXV1TFo0KDC+nHjxsXkyZOjf//+8corr8Rtt90WF198caxdu7bw/yLummuuiTPOOCP69OkTL730Utxyyy3x4osvxvLlyyMi4o033oiKiop2x6yoqIg33nijMObkk08u+vknPvGJKC0tLYw52nU0R9u2bYudO3fGvHnz4jvf+U5897vfjSeeeCImTZoUTz31VIwcOTLeeOONKC0tjU984hNF+zv55JOLfv/m6KPR1XmKcC19XPb2792fuvfee6O8vDwmTZpUWOda+nh1dZ4iXEsfl73N0Q9+8IO4/vrr49RTT42SkpI45phj4u67747q6uqIOPKuJRF7iM2aNSt++ctfxjPPPFO0fsqUKYXXgwYNigsvvDD69+8fS5cuLfyjcf311xeN+dSnPhUXXnhhrFu3LoYMGRIRf7wR/4OyLCtafyBjjmYdzVFbW1tEREycODHmzp0bERGDBw+O1atXx4IFCwpx1JGu/P7N0f59mHlyLX089vbv3Z+655574pprrokePXrsd3+upYPjw8yTa+njsbc5+sEPfhDPPfdcPProo9G/f/94+umn42tf+1qccsop8cUvfnGv+0v1WnI7wSF04403xqOPPhpPPfVUnHrqqfsce8opp0T//v1j06ZNex0zZMiQ6N69e2FMnz594re//W27cb/73e8K/wXVp0+fdv/V9NZbb8Uf/vCHdv+VdTTa2xz17t07SkpKYuDAgUXjq6qqCn/13qdPn9izZ0+89dZbRWO2bdtW9Ps3Rx/eh5mnjriWPnoH8u/dqlWrYsOGDfGVr3ylaL1r6ePzYeapI66lj97e5uidd96Jv/7rv47vf//7ccUVV8SnP/3pmDVrVkyZMiW+973vRcQReC19bHffUtDW1pbNnDkz69u3b7Zx48YD2mb79u1ZWVlZdu+99+51zK9+9auim7zfvzn7P//zPwtjnnvuuQ5vzn799dcLYx544IGj/gb6A5mjz372s+3+YOjKK6/Mrr766izL/v8b6BcvXlz4+euvv97hDfTmqGs+innqiGvpo9OZf++mTp2aXXDBBe3Wu5YOvo9injriWvro7G+OWlpasojIli1bVrT+L/7iL7JLL700y7Ij71oSsYfAX/7lX2b5fD5bsWJF0WNIdu3alWVZlr399tvZN7/5zWz16tXZK6+8kj311FPZZz/72axfv35Za2trlmV/fGzQ7bffnq1ZsyZ75ZVXsqVLl2bnnHNOdv7557d7TManP/3p7Nlnn82effbZ7LzzzuvwMRmXXHJJtm7duqyxsTE79dRTj/pHmexvjrIsy5YsWZJ17949W7hwYbZp06bszjvvzLp165atWrWqMOaGG27ITj311KyxsTFbt25ddvHFF3f4KBNz1DUfxTy5lg6uA5mjLPvj/wAfd9xx2V133dXhflxLB9dHMU+upYPrQOZo5MiR2bnnnps99dRT2W9+85usvr4+69GjR/ajH/2oMOZIupZE7CEQER0u9fX1WZZl2a5du7LRo0dnJ510Uta9e/fstNNOy6ZOnZo1NzcX9tHc3JxddNFF2QknnJCVlpZmZ511VjZ79ux2z5J98803s2uuuSYrLy/PysvLs2uuuSZ76623isZs3rw5Gz9+fHbsscdmJ5xwQjZr1qzs//7v/w72r+Gwtr85el9dXV32yU9+MuvRo0f2mc98puhZfFmWZe+88042a9as7IQTTsiOPfbY7PLLLy+axywzRx/GRzFPrqWD60Dn6Mc//nF27LHHtnum6PtcSwfXRzFPrqWD60DmaOvWrdm0adOyvn37Zj169MjOPvvs7J/+6Z+ytra2wpgj6VrKZVmWfbQ3KAAAwMHlD7sAAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5IhYAACSI2IBAEiOiAUAIDkiFgCA5Px/+stDv7Sfnq4AAAAASUVORK5CYII=",
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "plt.figure(figsize=(8, 8))\n",
- "plt.plot(peak_hist, step_hist)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "[540.0,\n",
- " 653.6842105263158,\n",
- " 767.3684210526316,\n",
- " 881.0526315789474,\n",
- " 994.7368421052631,\n",
- " 1108.421052631579,\n",
- " 1222.1052631578948,\n",
- " 1335.7894736842104,\n",
- " 1449.4736842105262,\n",
- " 1563.157894736842,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625,\n",
- " 26711.86572265625]"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "peak_hist"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.10.6 ('autoparallel': conda)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.6"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "cc0ad6865167fb9a52c12f0fd0c8203c9a7690797bfee612a871d56b9d2024ce"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/examples/tutorial/auto_parallel/bench_utils.py b/examples/tutorial/auto_parallel/bench_utils.py
index 365e07e21..d9d656b85 100644
--- a/examples/tutorial/auto_parallel/bench_utils.py
+++ b/examples/tutorial/auto_parallel/bench_utils.py
@@ -1,16 +1,33 @@
import time
+from copy import deepcopy
from functools import partial
from typing import Callable, Tuple
import numpy as np
import torch
+import torch.nn as nn
import torchvision.models as tm
+from transformers import GPT2Config, GPT2LMHeadModel
from colossalai.auto_parallel.checkpoint import CheckpointSolverRotor
from colossalai.fx import metainfo_trace
-def bench(gm: torch.fx.GraphModule, criterion: torch.nn.Module, data_gen: Callable, num_steps: int = 5):
+def bench(gm: torch.fx.GraphModule,
+ criterion: torch.nn.Module,
+ data_gen: Callable,
+ num_steps: int = 5) -> Tuple[int, int]:
+ """Benchmarking a given graph module
+
+ Args:
+ gm (torch.fx.GraphModule): The graph module to benchmark.
+ criterion (torch.nn.Module): Loss function.
+ data_gen (Callable): Data generator.
+ num_steps (int, optional): Number of test steps. Defaults to 5.
+
+ Returns:
+ Tuple[int, int]: peak memory in MB and step time in MS.
+ """
gm.train()
gm.cuda()
step_time = float('inf')
@@ -39,7 +56,8 @@ def bench(gm: torch.fx.GraphModule, criterion: torch.nn.Module, data_gen: Callab
del args, label, output, loss
gm.to("cpu")
torch.cuda.empty_cache()
- return (torch.cuda.max_memory_allocated(device="cuda") - cached) / 1024**2, step_time * 1.0e3
+ peak_mem = (torch.cuda.max_memory_allocated(device="cuda") - cached) / 1024**2
+ return peak_mem, step_time * 1.0e3
def bench_rotor(gm: torch.fx.GraphModule,
@@ -47,19 +65,92 @@ def bench_rotor(gm: torch.fx.GraphModule,
data_gen: Callable,
num_steps: int = 5,
sample_points: int = 20,
- free_memory: int = torch.cuda.mem_get_info()[0]):
+ free_memory: int = torch.cuda.mem_get_info()[0],
+ start_factor: int = 4) -> Tuple[np.array, list, list]:
+ """Auto Checkpoint Rotor Algorithm benchmarking
+ Benchmarks the Auto Checkpoint Rotor Algorithm for a given graph module and data.
+
+ Args:
+ gm (torch.fx.GraphModule): The graph module to benchmark.
+ criterion (torch.nn.Module): Loss function.
+ data_gen (Callable): Data generator.
+ num_steps (int, optional): Number of test steps. Defaults to 5.
+ sample_points (int, optional): Number of sample points. Defaults to 20.
+ free_memory (int, optional): Max memory budget in Byte. Defaults to torch.cuda.mem_get_info()[0].
+ start_factor (int, optional): Start memory budget factor for benchmark, the start memory budget
+ will be free_memory / start_factor. Defaults to 4.
+
+ Returns:
+ Tuple[np.array, list, list]: return budgets vector (MB), peak memory vector (MB), step time vector (MS).
+ """
peak_hist, step_hist = [], []
- for budget in np.linspace(free_memory // 5, free_memory, sample_points):
+ raw_graph = deepcopy(gm.graph)
+ for budget in np.linspace(free_memory // start_factor, free_memory, sample_points):
gm = metainfo_trace(gm, *data_gen()[0])
solver = CheckpointSolverRotor(gm.graph, free_memory=budget)
try:
- gm.graph = solver.solve()
- peak_memory, step_time = bench(gm,
- criterion,
- partial(data_gen, batch_size=2048, shape=(3, 224, 224)),
- num_steps=num_steps)
+ gm.graph = solver.solve(verbose=False)
+ peak_memory, step_time = bench(gm, criterion, data_gen, num_steps=num_steps)
except:
peak_memory, step_time = budget / 1024**2, float('inf')
peak_hist.append(peak_memory)
step_hist.append(step_time)
- return peak_hist, step_hist
+ gm.graph = deepcopy(raw_graph)
+ return np.linspace(free_memory // start_factor, free_memory, sample_points) / 1024**2, peak_hist, step_hist
+
+
+class GPTLMModel(nn.Module):
+ """
+ GPT Model
+ """
+
+ def __init__(self,
+ hidden_size=768,
+ num_layers=12,
+ num_attention_heads=12,
+ max_seq_len=1024,
+ vocab_size=50257,
+ checkpoint=False):
+ super().__init__()
+ self.checkpoint = checkpoint
+ self.model = GPT2LMHeadModel(
+ GPT2Config(n_embd=hidden_size,
+ n_layer=num_layers,
+ n_head=num_attention_heads,
+ n_positions=max_seq_len,
+ n_ctx=max_seq_len,
+ vocab_size=vocab_size))
+ if checkpoint:
+ self.model.gradient_checkpointing_enable()
+
+ def forward(self, input_ids, attention_mask):
+ # Only return lm_logits
+ return self.model(input_ids=input_ids, attention_mask=attention_mask, use_cache=not self.checkpoint)[0]
+
+
+class GPTLMLoss(nn.Module):
+ """
+ GPT Loss
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.loss_fn = nn.CrossEntropyLoss()
+
+ def forward(self, logits, labels):
+ shift_logits = logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ # Flatten the tokens
+ return self.loss_fn(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+
+def gpt2_medium(checkpoint=False):
+ return GPTLMModel(hidden_size=1024, num_layers=24, num_attention_heads=16, checkpoint=checkpoint)
+
+
+def gpt2_xl(checkpoint=False):
+ return GPTLMModel(hidden_size=1600, num_layers=48, num_attention_heads=32, checkpoint=checkpoint)
+
+
+def gpt2_6b(checkpoint=False):
+ return GPTLMModel(hidden_size=4096, num_layers=30, num_attention_heads=16, checkpoint=checkpoint)
diff --git a/examples/tutorial/auto_parallel/demo_gpt2_medium.py b/examples/tutorial/auto_parallel/demo_gpt2_medium.py
new file mode 100644
index 000000000..2739a4c2e
--- /dev/null
+++ b/examples/tutorial/auto_parallel/demo_gpt2_medium.py
@@ -0,0 +1,108 @@
+import time
+from argparse import ArgumentParser
+from functools import partial
+
+import matplotlib.pyplot as plt
+import torch
+import torch.multiprocessing as mp
+import torchvision.models as tm
+from bench_utils import GPTLMLoss, bench_rotor, gpt2_medium
+
+import colossalai
+from colossalai.auto_parallel.checkpoint import CheckpointSolverRotor
+from colossalai.fx import metainfo_trace, symbolic_trace
+from colossalai.utils import free_port
+
+
+def data_gen(batch_size, seq_len, vocab_size, device='cuda:0'):
+ """
+ Generate random data for benchmarking
+ """
+ input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), device=device)
+ attention_mask = torch.ones_like(input_ids, device=device)
+ return (input_ids, attention_mask), attention_mask
+
+
+def _gpt2_benchmark(rank, world_size, port, batch_size, num_steps, sample_points, free_memory, start_factor):
+ colossalai.launch(config={}, rank=rank, world_size=world_size, host='localhost', port=port, backend='nccl')
+ model = gpt2_medium()
+
+ # trace and benchmark
+ data, mask = data_gen(batch_size, 1024, 50257, device='meta')[0]
+ gm = symbolic_trace(model, meta_args={'input_ids': data, 'attention_mask': mask})
+ gm = metainfo_trace(gm, data, mask)
+ budgets, peak_hist, step_hist = bench_rotor(gm,
+ GPTLMLoss(),
+ partial(data_gen, batch_size=batch_size, seq_len=1024,
+ vocab_size=50257),
+ num_steps=num_steps,
+ sample_points=sample_points,
+ free_memory=free_memory,
+ start_factor=start_factor)
+
+ # print summary
+ print("==============test summary==============")
+ for budget, peak, step in zip(budgets, peak_hist, step_hist):
+ print(f'memory budget: {budget:.3f} MB, peak memory: {peak:.3f} MB, step time: {step:.3f} MS')
+
+ # plot valid results
+ fig, axs = plt.subplots(1, 2, figsize=(16, 8))
+ valid_idx = step_hist.index(next(step for step in step_hist if step != float("inf")))
+
+ # plot peak memory vs. budget memory
+ axs[0].plot(budgets[valid_idx:], peak_hist[valid_idx:])
+ axs[0].plot([budgets[valid_idx], budgets[-1]], [budgets[valid_idx], budgets[-1]], linestyle='--')
+ axs[0].set_xlabel("Budget Memory (MB)")
+ axs[0].set_ylabel("Peak Memory (MB)")
+ axs[0].set_title("Peak Memory vs. Budget Memory")
+
+ # plot relative step time vs. budget memory
+ axs[1].plot(peak_hist[valid_idx:], [step_time / step_hist[-1] for step_time in step_hist[valid_idx:]])
+ axs[1].plot([peak_hist[valid_idx], peak_hist[-1]], [1.0, 1.0], linestyle='--')
+ axs[1].set_xlabel("Peak Memory (MB)")
+ axs[1].set_ylabel("Relative Step Time")
+ axs[1].set_title("Step Time vs. Peak Memory")
+ axs[1].set_ylim(0.8, 1.5)
+
+ # save plot
+ fig.savefig("gpt2_benchmark.png")
+
+
+def gpt2_benchmark(batch_size, num_steps, sample_points, free_memory, start_factor):
+ world_size = 1
+ run_func_module = partial(_gpt2_benchmark,
+ world_size=world_size,
+ port=free_port(),
+ batch_size=batch_size,
+ num_steps=num_steps,
+ sample_points=sample_points,
+ free_memory=free_memory,
+ start_factor=start_factor)
+ mp.spawn(run_func_module, nprocs=world_size)
+
+
+if __name__ == "__main__":
+ parser = ArgumentParser("GPT2 medium Auto Activation Benchmark")
+ parser.add_argument("--batch_size", type=int, default=8, help="batch size for benchmark, default 8")
+ parser.add_argument("--num_steps", type=int, default=5, help="number of test steps for benchmark, default 5")
+ parser.add_argument(
+ "--sample_points",
+ type=int,
+ default=15,
+ help=
+ "number of sample points for benchmark from start memory budget to maximum memory budget (free_memory), default 15"
+ )
+ parser.add_argument("--free_memory",
+ type=int,
+ default=56000,
+ help="maximum memory budget in MB for benchmark, default 56000 MB")
+ parser.add_argument(
+ "--start_factor",
+ type=int,
+ default=10,
+ help=
+ "start memory budget factor for benchmark, the start memory budget will be free_memory / start_factor, default 10"
+ )
+ args = parser.parse_args()
+
+ gpt2_benchmark(args.batch_size, args.num_steps, args.sample_points, args.free_memory * 1024**2, args.start_factor)
diff --git a/examples/tutorial/auto_parallel/demo_resnet152.py b/examples/tutorial/auto_parallel/demo_resnet152.py
new file mode 100644
index 000000000..5861371e8
--- /dev/null
+++ b/examples/tutorial/auto_parallel/demo_resnet152.py
@@ -0,0 +1,74 @@
+import time
+from argparse import ArgumentParser
+from copy import deepcopy
+from functools import partial
+
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+import torch.multiprocessing as mp
+import torchvision.models as tm
+from bench_utils import bench
+
+import colossalai
+from colossalai.auto_parallel.checkpoint import CheckpointSolverRotor
+from colossalai.fx import metainfo_trace, symbolic_trace
+from colossalai.utils import free_port
+
+
+def data_gen(batch_size, shape, device='cuda'):
+ """
+ Generate random data for benchmarking
+ """
+ data = torch.empty(batch_size, *shape, device=device)
+ label = torch.empty(batch_size, dtype=torch.long, device=device).random_(1000)
+ return (data,), label
+
+
+def _resnet152_benchmark(rank, world_size, port, num_steps):
+ """Resnet152 benchmark
+ This benchmark test the through put of Resnet152 with our activation solver given the memory budget of 95% of
+ maximum GPU memory, and with the batch size of [512, 1024, 2048]
+ """
+ colossalai.launch(config={}, rank=rank, world_size=world_size, host='localhost', port=port, backend='nccl')
+ model = tm.resnet152()
+ gm = symbolic_trace(model)
+ raw_graph = deepcopy(gm.graph)
+ peak_mems, through_puts, batch_sizes = [], [], [512, 1024, 2048]
+ for batch_size in batch_sizes:
+ batch_size = int(batch_size)
+ gm = metainfo_trace(gm, torch.empty(batch_size, 3, 224, 224, device='meta'))
+ solver = CheckpointSolverRotor(gm.graph, free_memory=torch.cuda.mem_get_info()[0] * 0.95)
+ gm.graph = solver.solve()
+ peak_mem, step_time = bench(gm,
+ torch.nn.CrossEntropyLoss(),
+ partial(data_gen, batch_size=batch_size, shape=(3, 224, 224)),
+ num_steps=num_steps)
+ peak_mems.append(peak_mem)
+ through_puts.append(batch_size / step_time * 1.0e3)
+ gm.graph = deepcopy(raw_graph)
+
+ # print results
+ print("===============test summary================")
+ for batch_size, peak_mem, through_put in zip(batch_sizes, peak_mems, through_puts):
+ print(f'batch_size: {int(batch_size)}, peak memory: {peak_mem:.3f} MB, through put: {through_put:.3f} images/s')
+
+ plt.plot(batch_sizes, through_puts)
+ plt.xlabel("batch size")
+ plt.ylabel("through put (images/s)")
+ plt.title("Resnet152 benchmark")
+ plt.savefig("resnet152_benchmark.png")
+
+
+def resnet152_benchmark(num_steps):
+ world_size = 1
+ run_func_module = partial(_resnet152_benchmark, world_size=world_size, port=free_port(), num_steps=num_steps)
+ mp.spawn(run_func_module, nprocs=world_size)
+
+
+if __name__ == "__main__":
+ parser = ArgumentParser("ResNet152 Auto Activation Through Put Benchmark")
+ parser.add_argument("--num_steps", type=int, default=5, help="number of test steps for benchmark, default 5")
+ args = parser.parse_args()
+
+ resnet152_benchmark(args.num_steps)
diff --git a/examples/tutorial/auto_parallel/demo_resnet50.py b/examples/tutorial/auto_parallel/demo_resnet50.py
new file mode 100644
index 000000000..4cbd53eba
--- /dev/null
+++ b/examples/tutorial/auto_parallel/demo_resnet50.py
@@ -0,0 +1,107 @@
+import time
+from argparse import ArgumentParser
+from functools import partial
+
+import matplotlib.pyplot as plt
+import torch
+import torch.multiprocessing as mp
+import torchvision.models as tm
+from bench_utils import bench_rotor
+
+import colossalai
+from colossalai.auto_parallel.checkpoint import CheckpointSolverRotor
+from colossalai.fx import metainfo_trace, symbolic_trace
+from colossalai.utils import free_port
+
+
+def data_gen(batch_size, shape, device='cuda'):
+ """
+ Generate random data for benchmarking
+ """
+ data = torch.empty(batch_size, *shape, device=device)
+ label = torch.empty(batch_size, dtype=torch.long, device=device).random_(1000)
+ return (data,), label
+
+
+def _resnet50_benchmark(rank, world_size, port, batch_size, num_steps, sample_points, free_memory, start_factor):
+ colossalai.launch(config={}, rank=rank, world_size=world_size, host='localhost', port=port, backend='nccl')
+ model = tm.resnet50()
+
+ # trace and benchmark
+ gm = symbolic_trace(model)
+ gm = metainfo_trace(gm, torch.empty(batch_size, 3, 224, 224, device='meta'))
+ budgets, peak_hist, step_hist = bench_rotor(gm,
+ torch.nn.CrossEntropyLoss(),
+ partial(data_gen, batch_size=batch_size, shape=(3, 224, 224)),
+ num_steps=num_steps,
+ sample_points=sample_points,
+ free_memory=free_memory,
+ start_factor=start_factor)
+
+ # print summary
+ print("==============test summary==============")
+ for budget, peak, step in zip(budgets, peak_hist, step_hist):
+ print(f'memory budget: {budget:.3f} MB, peak memory: {peak:.3f} MB, step time: {step:.3f} MS')
+
+ # plot valid results
+ fig, axs = plt.subplots(1, 2, figsize=(16, 8))
+ valid_idx = step_hist.index(next(step for step in step_hist if step != float("inf")))
+
+ # plot peak memory vs. budget memory
+ axs[0].plot(budgets[valid_idx:], peak_hist[valid_idx:])
+ axs[0].plot([budgets[valid_idx], budgets[-1]], [budgets[valid_idx], budgets[-1]], linestyle='--')
+ axs[0].set_xlabel("Budget Memory (MB)")
+ axs[0].set_ylabel("Peak Memory (MB)")
+ axs[0].set_title("Peak Memory vs. Budget Memory")
+
+ # plot relative step time vs. budget memory
+ axs[1].plot(peak_hist[valid_idx:], [step_time / step_hist[-1] for step_time in step_hist[valid_idx:]])
+ axs[1].plot([peak_hist[valid_idx], peak_hist[-1]], [1.0, 1.0], linestyle='--')
+ axs[1].set_xlabel("Peak Memory (MB)")
+ axs[1].set_ylabel("Relative Step Time")
+ axs[1].set_title("Step Time vs. Peak Memory")
+ axs[1].set_ylim(0.8, 1.5)
+
+ # save plot
+ fig.savefig("resnet50_benchmark.png")
+
+
+def resnet50_benchmark(batch_size, num_steps, sample_points, free_memory, start_factor):
+ world_size = 1
+ run_func_module = partial(_resnet50_benchmark,
+ world_size=world_size,
+ port=free_port(),
+ batch_size=batch_size,
+ num_steps=num_steps,
+ sample_points=sample_points,
+ free_memory=free_memory,
+ start_factor=start_factor)
+ mp.spawn(run_func_module, nprocs=world_size)
+
+
+if __name__ == "__main__":
+ parser = ArgumentParser("ResNet50 Auto Activation Benchmark")
+ parser.add_argument("--batch_size", type=int, default=128, help="batch size for benchmark, default 128")
+ parser.add_argument("--num_steps", type=int, default=5, help="number of test steps for benchmark, default 5")
+ parser.add_argument(
+ "--sample_points",
+ type=int,
+ default=15,
+ help=
+ "number of sample points for benchmark from start memory budget to maximum memory budget (free_memory), default 15"
+ )
+ parser.add_argument("--free_memory",
+ type=int,
+ default=11000,
+ help="maximum memory budget in MB for benchmark, default 11000 MB")
+ parser.add_argument(
+ "--start_factor",
+ type=int,
+ default=4,
+ help=
+ "start memory budget factor for benchmark, the start memory budget will be free_memory / start_factor, default 4"
+ )
+ args = parser.parse_args()
+
+ resnet50_benchmark(args.batch_size, args.num_steps, args.sample_points, args.free_memory * 1024**2,
+ args.start_factor)
diff --git a/examples/tutorial/auto_parallel/imgs/gpt2_benchmark.png b/examples/tutorial/auto_parallel/imgs/gpt2_benchmark.png
new file mode 100644
index 0000000000000000000000000000000000000000..eec121758149b516cd5437cdb0df140b753f26cc
GIT binary patch
literal 66851
zcmeFZWmuKp*Dkv3mY)h@5h@|AQX&!t!Xjl+(u#C9(x^W{K?D{lDX{30Zcq_vQBqpz
zF6rE3KKMW9eXp}WoDcikez`6#Vm&dRImfuiJ?=5)^ZJ3b=&{4+4&!jRV|cN91RQR^
zFb=ne_TYZ_kK@O575KqtE%MM>&P>xv`nGv4Iw?t&XLYfte{MI}iJH
zR$6^)Yjdlc92_S9&jIXamUzDZzxBvW(^SHZf?|=Tdap5rSe}4b?`vDs7&ew5|-3R~kyZH@r;s5;Z
zRuPTc{hvQB9RHtN{Le(;{>P$lxc{$Tn9~Y4F3IN^chtXqCYh?&7|AT3=-Y7(d`I%O
zX7w$91}S2t){mvBu7_EbQ!*UZ132==b6n|NQcZZk0sqhsPD#
znGXjC2T9EdG6_Fv&Yoo=BO_C0Jzcue@5Mk@A3D9k3V*GxuczF8v0tUgI;mrBD{#+&
zNCf_-n4+jE2qV_uKo9oDL{1E}kE0%=qek-e_U;M@^^qnQM2B
zoMydBKM>)ZRJ@t0oc%Sx>1;!dKcjysyJCvM?c28vyYkX(XL?O+OXSnw<4C*lmZYS-
zB!!e9*3(|KL2O~)#&9@YOCEk4?nc&fpR4A}Bb17gl5cp7f4;XE_&V0E>DpT6ygoQK
zW&oEcEG#tm`sS>|fHyDEB#PH8691ZZb5r^H(YuuFYTrqC|CUr`PBxVsG5Eg9-+PzO
z{Fj?M8BO}y;)GBOMUWHEE?o6P9@Fl`{m0H-xpGC!d3{MiSlCTEN-(V)ni(;x+W~ud9)7o_LjCOWM6Q_oIYp#(J
zw^5tq>({SuKAn9JbCVD!5p4KpxFPvL^z)86!qD_|`dD+q#8^UMQnFZ
zkCga++|?rhhhlf7!gv${nHA{p&T9*@-@^G8D=9Vge}8^C-kFm#AceCCSW$cNlY7~;?nL3Q*Qxw^94O_(rzIr$R
zd+3y;q^(^;D0eeQ`9@4#Fb9iLy4undSd&uZEqnQrrxxnPwhdqY-9El4f?HV**?k1p
z^y8Gwt&8}Z7XAM^ceNzRDHK}j*gt)GMfB~3@uoNlPXm{>49$$URQ${^^`@q#hmRg@
zNy#Wn9vvNB7aX4MDGubfGi}Y#)Z~SsnXQ}l6d5c}_uRA|^>yxA{WAhXW44)SV{3h)
z_2db+_2>f%7RCC=8E@X+2gb(6S+@BJS3)`U-qr>%T?SKcO;wh2T%Hc3lM1EF+XQJ3FgwVw&;Na`NXjw2-~My$<6^30!c4!h(WP
z7&z0n^~vnU+QvqiYuBzN305@tC))3wv+BNcRcq3@6SNrRrcu9?m#wG
z#p&s3X4SmZ;l`+_@h9qqmLfgHcELVAJ{ubwd690*l!sp#>?&T&0WWMW(~j^FCoi~Gn5|CBQ~H}~xKpaW;`GE-$eEg~Xf
zFh5wc)VB%O%$b34wR1M!ioIn&U)|6k_36{6wYe%9|A-k|A-%R#Wry{te2Rbmp#@t*
zE6~bxeLIc69aq@nvj(%mWjpzhf{Ac1GBVPziS!ygiLK)5LizTl%)Z0qz8)SPD*2{q
zDM}gfH)dca6%wSW?CtGqy1G=s-`ck}*Js}8tS?PvJ!B2NWpC-}To=UV4;a^9=|zv{
zHSPY+l2HC+&@_0(;--!G=GvlM4UvB6_wVR%KC9#>Rd30-sK#@R+U3(!xGAnY5cVLWDKlGdOH-|b
zi-Sqv^T7MY0G|9)6^)Xs1gh<_05(#LJF?dPGzuwXXb9LlI3z#J)LL5|iK53FwWljA
zj<=%g6X_)khiU_vsP)>@)eSpylAN5J0@ze}xQ%~42;()AZW~@AFF?nayKD~5%w+7|
zcUT`zg@;aB>PV1@V)v?y0SM?>=HBg=>JBW56br+}6D^19nV80s3#kPh5}(fXuK~O-
z)UsrZ0!%9P6xrBpuGy?xz{r23spc{5an%jN2$EOQ$Y=xFIY--z>YMC0T*Rd8tFsup=w(z7KU8
zk4fif9Y?wR$hQc&9L;iaY}Qv63Se#6o*pr|NF@FH@7|aH{#)JGry1qE^bYw9_&&?U
zi?_3N8^qgq*x99pwl}ij|E}$i=U^>S)SK+cmYWO{aLfzkGKhgcSm3CO7cX{}*k|uQ
zdgeVEMTcHg>)b$_P_if9_dcgmXc?#BG$#W05(mVR14LhJ
zRnm;R(e6Rxl1MKRG}^%1*CN4DN&@6dl#RVlX8riP>ot>3F6TcDyeM*^^+{7NR)k4w
zDxN7z>2q0^GAmn+k91yAMq%;A{$t{E8(@>^VAf*;K0;
zL_Epyx!Zh3ZLB(tk?JtVMAArQ`rqwjKSeL1LJ?xHSpYwq7X(;h=1)2=)d6!pLaV=~IOR2qf0OI)@$?GV=I
z!+8?}>Sa#K=x}zmg7@H~$=Wr<<%ZF*u_W+MBZ&A*CXP#!KM-Cxe5!xp!st&td>tO>
z0l3j(4Pz9-E%hSn9}p;8;v^^$jMF$T_{GG;pfKK3U{T>R2`3}?nd-KXK|vv9dLv&h=RlM3pd_hyz4%imfuSN0HHW@CeffdK2Dz>=9q#_MIJ6(GGnwC
zr~Gyo?&%pF3yx)ZYHA`{Og`d5o_Dg1iB*JP!e9
zx#1RsqPs+ku_pQ_F!T2_wJImqlNHkp7sgxbt{$!M)n1tC>nk&Zz~}Do@1N|&4%c_emy_wLU;4nvGh3cO@6NfOXVM!+jv=U7VRz=m>xWLhK6%vd!lYb=zRb`
zf0M2vn-sVs(43v`aV=ZJ4I%GPSj*sfjs@wERn
zBOyHdafSPR2;W5KUSpoHIe5tPm0pwpXPkQ9_LTPK&viJ>wjaj1E#$zJDhzYaDWf1m*)X*U^>9#sBhS9Z*6)PZ}QP`ad9bYj@k+_+f8@ZtV!js?-{BK=ItLCnCdFF
zoiZ#aD(YOn3nA<(gzq=NU>O|4|BN)AFDol!%Ih-#*5K~t^XJ2g#>U2{
z0GpFtK-T;hUu=1GHQKMt{{9MF@(nuAs4evxvtlaP+wivtqo2k+u@DjNi;2B@oZT^5
z)mvm!cSWt>!;b=s0qw5o957mKXZ?zm;=W=aegoZwR`<=#Kgg2AP9Te1Zg5)vb4~Bt
zg(q|UmB<1=fBuZt47hgB6$0ti<1_BE5biGDxbf4ayTXYP!gu7wnh-(fBImB?yBuC(OqBCsSAo5Wyv>a{%USx!wu9i-d
z#;}HXaFj7vqopXISQH!@YA~+6sH%#J3KxE6X7OMS%{DN1_KK@H5NpYOiX9dWAXuu4Pj=)(6DN0F%xEug0@(zuszh9DfIV_Auv#I9&
z%y{HaOb-EqD-Rwsy;@{F-L1@TJ1J&632ubo9}qx-2x|?OlrYa=>7;J^nfWAFtMYK#
z!^}H!3DODx)h)mcTyt?NFey2X%cc;QI{%0h0EiXhCEsbAxUNs}0z*xN=(=PgEiFCX
zSMHi+tCXS9nyiol^qUdk-WFeh<&Y;VfHj~EdGqbf^H>se;~cMPDsT=+#1uQT^?U?Z
z2T#FdU2)yosEPkra=zj!m)*2lu5ky!{W0CTGu6qHF|S^|BA2_dIQrvq$IQ@is+(7U
zq^~WuYtpR#Sx9cJ#ll=glK_tx**xUFPKjp`xhYyR}ZZA>ReEj%PKEKbUEWf>1%w*INnJ_qs
zc2{^rgu;8_XXGaQcGIa!8ulW<<(eS~ke@ga4FEdxJX9fV;fBZrsN7>39I8zQt2qKgM$Rf(Pgi;M#lj&9WmV$Gl0VAT5tN-Ip
zdU`sUQ+ij1j$u?O5NU?>OAg~LOhXM}h||!uqjxYyz9Dy?V5Cx!2KQrgWL$-0ell
z!3?%a7S74Nlhi{zOonM;0$PZe!p7>K#i3x$WPq5?Qb#*aFP5z_$Ull)HjBzOmsFi0
zr%B2y{Ik3mu}KN+8~CxncZZYPmE}{6U=H_))18`yJsu@t05>T>ib8=XJ1kGLF^!S+
zLf{d%{U#&0dM`RU8hJQGKS(w?G0&jsqG`)+6ifkssI95NgWViHemqa}q0PE|91q*L
z2@uoTNP*&**jR3$pHksBWPktuef{WV_cX>iSlTbHm_@H_N}`dMM}C|E#>f$Bh6JbuHmUYz(vy|1sn}Q9#JWCD%5ZoqxZhR
zM?iVjB3a)y3Qgy@^xp$JMh*b8Up7HgbL!M7nwlrQbyJPM-`{>QX(ih?ChN>JfC*d!
zga$6rMB_M_$yKxdVar75_mWlkbsh0!
z*Lqi9uRwUxR5GEB2M~XP;K5y1hbV2PI=MSC!Tt&G&E>k~#HLxM1_Dx&;MU3$h#E40
zyrnZwNb%t4h9Ej6ubT>G$Ovu9Q!vYmzmbfpFOy@Zz16fA?g2!0RTke8^F?Zrzzwfkj<5IffA_g6`Aoc
z^HS>)XdcL~C)(0l!T;IRi{t^eT}PcD7ez1uF^*5_-Swf!#wU%D0w_le)rfO(99{oi
z;UTAX+YacA$GLNTv7%(TradZ<5;Ox(kq0)bt)$u`0VoCOfDo0S(?duDi3te_DtX55
zVn7EOl22FTD_x&1MkxfWQXE9$i5FyoPk)6k`}_LR;*0G6#FI!u?dD->X$kT*x5&TB*
zRH6KK>7b{?R##VRH(%`uf_MNEBLfLUGr*=o5`o@%<+sOn5zdk$KME9`M(3z@NP$wn
ze}CW%X1`&ZP*AXA-6*>QG^?N5QyZI`Ajmj_Jb1B#hTpWga4H5n9mqv`HK5GXV~|7>J$n|FcllFsv*
z{+!tcZ2$m4Ep)?gTlV0gLw=y&-G6f?dSzvWd|l4*tsSBf2ojaEbqzpVs}1AL01{)=
z7Znl`qQ%Svvj^zMq1-nJZVbxj_shx&>6Sya%qlsFg6or<>ya}<2H4Gg2T)N#2u&++?=_I6;ap#rFcIp)7VYd7br
z7nt8hTJBV5?hHRwrwOGS##(4oHR~v6U0qv4YT`Koha4zj03IZRlP84ms}k?Oy^sW{N(#U;lEADXx$~wL
zPys=;rtC`|NKHV@n~gF`;{i;-RS0XqP|!j6nvCHK{I;o}kEOw+Cjp^nhBORmA+Q!>
z_3tN
z-sgD@!O&1C2|}BWnJS3#X=;T8O{ck+U}FhD5>nuUwb59g9LShJ=Tt{kk$2a1(9ov3
z3*`VNa+di_>p*fwBh)B$7=x2wsx#zMA6{FW>f$Yp+S&uO8fBj?z_y^8fCn{37JP&P
zTo;6^M8z}}GA?IX0C{fh>VJ?vZa33g6W?57G2nif{K|V+=R~K~c~lSrw@-rc)1SAw
zrdxmJ7>yvx^R|>Of7z}9M%CxKT^h<`k_-d~a9sjSPYLV{?17Djh6YJo#FiM5BM^o}
zAk$`+i+=+u9SLBE8C02Nr#C3b0tbUssl24LlxFR-FF;%hu=i!-KM*mgUy=!~2t!fi
z0hlKWM|}MJMxXyZFTVC-G)Ba0azd;C9vq&{VXl%=q0DL3F|x0!N)!|cIasvT
z0*fbUn&n%59B)UUYzOr=q`kHvXo07!Yi^bUW$p+3YeUR~xIEb*2mBbV+QM*{SuKz=
zxNa*@Bvv7za*vxzKGXtSFnvgZ23qW#_ylo!n1@;S@A$4k>{awKg;xQ_LfbSy7J!=D!8bolAi}jYG*enZH;gYz!`bD*8loSrv)T|ff
z91p~86jcl%yGbc1xUp~Fz9f*X9hQ6S{5OwpApIOl7UMm3eF`9Rf%pT-sLfD~zbMeB
z*7ce444A4h2qUNlM{(go_;`zmS@~w(*2W(h#4$n5c>vM?l0fOgk~w3?Ap1lvUqU0NaaR`wg!K*Z3l&shmH2XTvCZ-LQ39-IL1Zq!?1pVeUua!yFu>hSr}
z+0RD|<^t&OKpNTs8nNmL(6b-F6ru3)9-McHzrbMt9u*Mn_mfel0({l#W+#E8hu*YFhDyoSgAt_iXX!Pv
zhczsTaRSU|o!QnAfw*@)3Cco=05)qtGN<`isgqPcJbKodXOeR4JP-0@-71SKDC-h4
zfQ3cS3fc3M@_DN2?p5Ruac-&oI8jK12g+`o^Kx_qp>Y8+5+hK^JYT<-1(wPMBSm-B6}7^i9)^hu
zrKrvseP^FkC|Awo4V*92hy7R=Om`J<&IOo2Lny$3C4@s+F9WcsJ9l$x@$fX_%KP^2
zB|V1TA{;J1^Y2Nb^6~+-!5kS-&Jb*|0Q%a7NU0Ebfk6Oj&G{IC>a6fJ9Sag4l$yrw
z(u~Z^aDdN?(xCf(be6^Kn*OYtJqF3s{L)eosC)d!;s19zk0Lel0ie#{;Fn)!oDeT>
z!-S!VBgz^e;P&L27#pX85l=0f@PvU#n+)#UM6(Zvdv|hEHpF}PL32uIN>BiO4r@x-
z%7(y*>YTb&)r9%sh7{oCDWF3ju!Is-23Wfh%sPs6$yIN_e=0o4WJX9ZN&}y*EA^i*
zjebP{p;T3mAF>9mvpu(YHI!-({8@ed%r2~FY1!h>uoRVmLyOBc
zq+=i@{{}>FJ|#A@g6tc!IuuB7;;Y2`c(btgQ8ALZDt`lrDvSuBaG-+D8?&qa*}h
zNn2Pzm^3eu@6U=~y?GOmk&yvU9%6MC9$S1t?>cX<5Zt5VN7S>9hkO1nZq#yHAh!|m
z^x@$LlZAGYTUeu-u*2)u0r5G0@j3o68DSCo(MK0TvpZ1Vnj7my!TX^dQa85~2H!~(
z*@(=~a`IE4BW|#QDZ!UPL(r9mDYghTjzR|snZ`iN)7*a#;M%#m%JriLb16;HmBdzs
z@tf+kN}nd>(TTriclzfH3|bE~eXAi>;tjW7$QPt{yhfkWO1#H<;z)sElyOr+bUmh9
zFXiI{3+=HFk30sA5DrSy{u3Oj%s{1iSz%1a_8j`Gny_&T_>ixt^xB>%iytNwYC3;oY+d*{@r58eG1GAb`AtCNCa>l?6Pi#=odfDZAN`?OVus2a@=58j>x8Q*p!8Plg?ll
zS|f-}BG@Aq1Q%Q}aG+!o6lW%A8uH(^%nFS_{i8T@8mtx4&Cz0!sNzDuPk`Mfl9{jg
zT?7^;W^v&Sn0ISGjSH2a54$#nnV>ZE1|_QVDMfrU%ViDdiHX{0nzHi!
z@4MXKlZbF3?oN5PGuBy-9b2I@*c@pJvcTtez474hmC;t-K>Eqns6IEF(Cw0lV
z&mxZw@I?j0UN4(BD#}#It-_Lw9
zgSItZOHViDH5-Jm7V=R1scU_Ay8~Ur_cZnpAAdWQ^>cWuT1`fpJZEE6s6829m{&9w
zz#PX|x9=~Wa^2av^3d$+`8CrX3p^pBzi`k{O8L>O4_UkFh@fA_RFU{#ZBkmtIg*_u
zo|x@l8Bm4b!7bGdYgfJC-k>`=wdxr7h?D$exag@OC49%|SNk!KUgA^{`D{q!#A->T
z)iQoL6??p#%BPO-%Z)ehx}=S&l8ojpa}6J+{0cZ0lOZA3iEo#|cN{@Dam58Y{4UB3
zpxe=W?0%}Cio*lqf{H(}c4ES>_M)V+PJDWeeTg4()PQb
z!^Suf;pL?oPAVCq*RxCNl4N?H%6EI`cz2QS@QZI!Cpz6rPG3Zj=@7JYziOvu=_j^9>KZR~CKg3?2=fteqM0%Nf`ZL~eTV
zK68hp;i^s3?oo6HH`blz9hAB#tv*x~ZWvzm_;M@RG=*!OtXR!dQ8qPRgH!ILJ#lvG;r&=N3h|Wa
zhtqrwzQC+U?1NbsFeI0MDO_G|H_XN%kS%Fh$6&t_=YY8r?Zgg{UH3L?yU_f)PIi9T
zw%O<+ecVb2$wtVuPC;`1_*D*CG!EK|zmNKRv@+~m`EUrG!NZhCs|629g{5&cp<<%0
zynIyHfw7qTx*ctVLJFsAM*m3@!w1*%L-&vP1(h&UsjznR)2DcKoWN}4?9rX&d}?Lq
zZdCU=#nR5AX+5p3HfwEC&FlBesf=~R3B8`HJdeiP;vY|}Nh?b2y((Z$goE{L)->_1bC54A5^<;e`4utK5Tv)I6v>sPcSRF0~AXxbrtM$N!L
z1o3Kj4xkCv`7~`lCFpRUvqw<2_g_w;+53fsTQwDpllmX`@I#9th{|jrGiZGKMz)m8
z!o#D8l-hMaKR*Ks1oPD=G3M)bbYLy*hI7fNjFkE6ogvmL`o?g=@+#5HB|CN(87rjU
zNc4w9T_NV)>jQ$OAPqx}B6oZ5{{8Wf#o;<1=aiwPp#JzS?R#)-_)Ov6y?;Rttbe-)
zXu;K$5%uKKkz?HXrh&iv=_MLP2wSzfRkb;zjih5xj|0IY1y!e~p{uM2<)|sGhJCu(
z9olEyQx#K!I5g84BcEQXv`F&C9Xfh+a1-YH-D#AgV8p=pqOpL5f`@Ht>v}rBww0xy
zT(AMTkS`PDnfg%IsL4a^QjqWjx@@iLQLwAsXdQrZ7F6YfsF6Xtu|C6$RFX3IQ8WLa
z&atYR!jz2tW4gR_&fd**0i+oikC;KvZluqxkj$A0#43fn~XMOHF8V!9P4a
z9CdPQSFNwF*Y)%WRANlWUmP*hdCZu%uxoPK_^`-AWw#8lx
zrPT*S)bk1bp<~dz0o5-~7HXJPH+e`e%_-|;58{N^3UUmJVqDIrQ(KrBK1%o}H>vvh
zh|l58clF-Ou#(!g3wn=u7%@|%ow)vXD}AzVf1>YD$^N0vm^O;}cT|Vqg1BF_SKNO2
zo#QbMVLdI86$NQ{$8r^vm|eHGYTt1tpei`X-4h`A)qFbvVjk$5NQ#1t)M0b6JH&o2%fB{z#V<(xC6I$?e$TEMir>5=xuj=c>Kb-5A_F&c%3!?MFxrnY2P9Rki6Ql
z%*eoi>b;&`Ud^CQaof*pLk5n&>wYvdpAY(vK?i`TJf{!+!wy4%O3+C`1NDshUR{tv
z&)qP8?|z&rIfCCV`L75D3U2d=2u#Pa?HET#yp6AwR!$pzP;<>q&BPBvcl$A<@i}hv
zcd7t9-a`jM!cR)5jKlgEp~lhg(J$ywEe0w4NOXiUD=MfO{)l;rMh#_tG2&Mrnhf4P
zCpplTiBQ7mb^0U4rkr&TdQ7sTia~or^=TJq>%bF0+*HUlRPa8_d7lWK=H$?cwrkIR
z)R^Y*e&q9om%wNJUU4>&T_g;ioyw#5Q7lMfIL&g=(kd@BqW<^ysG9Dl+rUnBMP2vn
zKG}Pay702-+bf!7PDogebQouP`SK;6h?){tR-QuRO?7V`EaBn9hnoPJ=ZnN>dUB!lgAeJtsKyJOQPx`<_Ju`7&|l5baSiH?
zKfwFx@z8lCqqtE}y1i89nEU$88+lOd($$OOq5na?>jFry=^am=Jb{X_Q90C$R?TI=
zKW_Q13~L(H{a|qz&{g72R=eNoHOR$(65093U9z}Oms@iTl$kYbVJISG28V~>c4!;v7%S&f8OaS=?p;K6`&=Qb7zH@kq
zkH86q6p?%Pe!4=velsJI6C%fp28IS#jZQ(?gpA&%#J+7JU1?8hPL^X~=mjiE_?39Y
zjaI6s#9j_{b%TnCI_y!CI>;^gGeS-)AE1*<9x8=+0yrhwxuIV2^hd8Q6cy*^=43(B
zpETLBf7@kZ8dcHSssI9_blX?ZiqPr#cyG{iVVuNGz?tppt0bKlaYA~vD~mSXYxXa-
z`c0B$={!1>YRznKq6u}|6N`H;3hLi&BO#IlWf
z2`D}$K`jLIECpz^LsIQJsy3o3))9`9_W*lHcZV(zhwaVfDHEgsp(b4@mA0aS)O+Dw
z&hP;^P-Z@$cs6R(K)pQBn}RBo;OF_|M`HKj+_-X_V{*p#;0cnkCmc*oPT(^gqFrPu
z8X_9Hjj`c+=v{+g%D>n5gvkB-sM{S%7?K|Qa33pfes~btEO*_Q(9^!Cp?zMe!%1Ak
z(tfNxc%r(e($MxZ>$5G%;jin!46gI9ab7(y_Xwcu*N-A6`Oq%Lq!ro11Y3%!gd|mI
zJW)_zHK-pB1nQxFb>C_DNu3rh66athKC*OWtEQ3b=_JEh_m*^mi+~1Ka0hJLfiLB~
z7)`sSa@tGY&R09zpx^z}P3tZ9ja%vq(%DVAj2IKrz9{(&Sg_kR`+RuL`*L!}in1}u
zdvD6rEbb8(Y@3^E-!(L(V(GLS>nkivUonfxz{_>vDaphR{UCVI=$vtPXYbA!UAd>g
z5`LE|h5RZ%J-R=w*JCW@-zqIoZCYT*SNa(5B2+{ZNjmF|6U%oghqQ9_?{#HCt3AU3
z7y*PuZY=khouT_GVdWs6z$-Cb29gD2JVhr?Wki=x2Ga>>W+Zy9I);UvLMQmrV>$81
zitG-O=6*5JApEy!LVgD!@)5piq;9+EW%Waf`mv;~W1jCWkYgt({5?UH^G+w{-K>Xk
zZ~f!mpCgh45;BbDz2+$u
zaLqV+*HHZ;>{)3vu@pl`3o8!qXeQcsb`6ns@zdp(w=aj3j0G%3c?b9uXZMPj$N@>a>x667jO^&P4e-D3yMu6Mcb6FUEn466(i*puszmxKby~d`I;0bD9Zg6^p@5W(qNW^b#5V<=jsiR-JhWpZOX4oj(7pO=b#FQD~
z@rQd+YQ&_h9yj@tc>kGo6o>tRpLSHGPvx5P|23HY#+h}N;8iuB?1FdwlZBir0_D4xAQ?
zQkIk(d#1_}ti!`ttf8gYnTNsU
z<+x*-)r!9jHWe4trMfJ|T`!bs^-=vANM~=|-CAh)etylNdjM0ca3Ma}RFI^ez&T<>
z!Hs5BA>&>qJmjbj^p+Dz3OM1Fa>qi~=4#pE*M3T=y
z@u~;IW&-$2jB+P(W0Iio;&RLc9b`hz*lko~3?8!05jglwyEc1TmaE$mHwhZwv
z)=Zh$N@5m3eF&TBvhGrrH;jf%=?uc$Y}_@drt6MT%sEAKeXcA8rypJ|P#vZnLu)
z*KJRs^<_mX+wivfv;+#<#UN+L);k@oX%4=jr)~X;bv7M9&9U!P#wdN^59Flz8X$H_Q*=eSc7bnV7X
zazdPI7#9B(rnPsU$nWR2(`(ZAiKR5_%nz=cNil7G(UR;WXkQrDl*@V4pvI`_*-IpA
z-%_a6WTNuNPJeV9Tb8pwqx8~^=D*6RY>X&ZGSGQ1&06OsE37(=D+-ADP|OxDA8LCO
zU@IA*?bqqdLSf=N{0@8a>VrEo_h4eHh!lA$zcwf*zEFJM(}|>W`g2-+7HcW>{t
z$S3V7N22&U@@}4~&pUJe3Df+l-C_o?JRe+hlUTS&sOV2VGs@^yIsc2%VllvBlbLEu
zA$i5R^CbrNw0kfG&Ucr$hRwZ%jQcDZ8O2|%Q%J^1X)_kQvcdEjYWREss|OpHF=*qZ
zbx{Nqw)AN1`<_4XR)XlBf7qpd`(TL?GB(-|Js?Ve}YZFjK{_upvUhC3y>ES@;Sgi+ty0z7u7BZNcp$L
zw51~fa^AnA#g%MGt<
zCX_Y>li%E${5$OhYyz8h{z&22YL@&(7fC*!BXSPyi7b-;y-+L$E@qIwF>!Fu_N04auOV0eH!h^D_ATMx*8C0
zhB45To_kxZs81i$cyL)aFgF@A-t?1D`LLcE&e~LMJ2YrqDjHmmDo!kwJpqT?13O(g
z_>`MDSa+s}99t@D@>5ZXc9fI;bc>0~JHEC{S~00vhEStR2p%J5t9E*z$PX#cjW877
zXj#q{uVLctEkRRxW3$bfYf^x>JiOyH;eA973P!EM5OPC
z)8^zD(lvehZ_=%@GHXzPw=s{wZ?7gl9b&dei$NAtv$h^%lWC6IIp^n47@GWxe;N(L!(kn{pc}zC4jJ)KvQzpC9IQMPW2FeP?aW?|s8S!Cd~@
z+-nEDEe$*YUX-D5@jJ(&yU4i2rzz6pzPKt)$$Telm2O)%oDmtxn{y&x(Ov9%WkCeCgvIf5HGZ60r1c|gHv7w?w1_{`~CM6!HvcD>3yC`VgZXI!?=WtD0cdFs{vERWob8~EIsr|e~IgEP0nJcq4y~@)W(%Rfk4|**2XH7g|
z%eUK!c;994ik9g^Y@RM$z^q=B7QqcGz|FWu?0e?@I@^v3m6rl1=n7ZGL1(w8;bfwB
zb2S2061BB}JPRQq270Kvvp1o=0qtxbs(&X(O-+runIO5VTR3zCt}jgdTZ0t6YT9ru
zqTm9(s~PXH!%Xk8Zog9`{4OGTXcp=1neBru(@CHep|-vjP$AaymgncO7DA*YXlQ73
zf>;&lIL!x}o5!FPQp)2C8a!rb8=|5dy)6aYP!qYUy1%Zfhu)wn-2g8Fm`YkF7jlzK
z`P<5|Cm-6ZW6&nAa!qJvM1A5D?HLJ(n&C*N;y=RA6*36>AW1T_|A^mDZ5=TlH$GRAhc1Tfp&{?rII@n2+ptxPdj
zHviKL5{3`*A_m`8bo=5Yo_>IKhDjl40+FsF!Na-@JcgbjoSUn`my828Q;H>sAiV2*8F%CWX)J>DuYS_g
zS1k^22EQya2rDWUm_NoNvC8`Q=Y7ziBndkYrE=3i%K%Nci4&=&zrL^Bhp;`%VSLk8
zL}=#5WiOF)Khne1F3Cthih1n>;gLevS8jm>-#m0FMc36HlV9Z
zD#L?r(?YJ-o}?i`S{Wr>I*9Yga$dLP*8l#|#r4<6XG9Wokt697S}WxqJ`8a#Z_sUq
zTiU?xN@!k8fxdp!FuK@n)fne!i*AyVb;kX12-WWDSC=kd9)dpeSFa9&G9E&80Ox!A
zLb0`+RJjV`lVxLS_ZzaP0-*rRf7+LMjq~0487PrdPmVTzG)_hBF|%&})fz1m$24c~
z%{RqV8A7N)5QML8Oi2H@Cp4c^EzCy~l3tNm=lEJnb3bjHK+E9k{8&!|u3!)5;s?yZ
zhG~g|~chKWwwI3(Nn6`6k+(o(!bR
zJzf%+li6uS=()`hzjLMKfgP^%#`tsRz)cq>@xm-{h(?(>&W5;{WAq?g>tomYMU53r
z%qgoldRjiWx>ieBoa@Yb<0PO)>i4E+EZRY2ZELKkFZS~BnYH;5g-|W3^z$04@-Zt~
z*_W}v=jMUcerYKrcxIaGMtbE16H+{v>o0me=G+l6aZeA{_rV>_J}#H4p|f}++_@GmK9Mc8EP|%9#sAKS84xs5ku|*u$7^t
zQu+R2o}TO7A!5H(g2I!kZUt9F2
zZv!1$aqZhOXnr1YF?rBJcEBJ8WV2&1hOJu?w2zGx&mw+{ka701(!Qyr7a%{4i7I}=f^rAa8LK5LAj-l
z%%3n&Ilwd>f80@#P#!`l&+6l|*2bh|dmRlOH~R|9Hk2g>l~LPL6@Xim{l)wmu?Va2BXPT52RRrn*7i3eEz|Pm&>&p
z#SWozq-V%Rqzp*nMFEA<_dESG5aqEv`9|57y)`w=%B)4!b@6Sk+|3;;XIhyeC1!Xy
zOm@UqobUdSQE;9cJYiBQKV?+rL$$O97P&vmVb*ye
z__0`BU*J|aeT7zQL7hS_XT7N4^tUy^98V_!^?GR_$ne#4F!&Z2e^u^p34=cm@lDCZ
zyWHcxKB>~Do&U(|P}D>;44PvH(z0PD^wywCdu;hPkyJ5sAf+)4t*
z<%_s6tk1&y)&WYqm!=<`deTqLDO&~K$6_+=2MNoeFD1P(5{hd&fad4d!vS1r>P|$-Ha3sMtWF!I?ng70%H30H@ONqS|ya=Qg-d+K(GAM$MS7I(~=I{ny
zK?lk;+HVSF4)M8_|4jvKj%ye0>?iL(knlC-f)F^<9`zk++k@JEUu{!JT3a3U4cN$W
zO;r2-A-x7M-2j@;9VRoYh$gO}e8YapOAYThMZx=9VOIof4dP$lLRMd2xD4F+!I35dC@Q$u}XaYxDg+N`;wHlE|&EN2@
zhoYx*TAf1nP+jyaW0x}Fpi-r8aJ`*!JhJ^2%0Opwuf!J7TZV;UvJNifvlhe>+k-cP
zVEbJ%)DG967k%N=0UE4(7$Q8{Ir@L7J|?6KMKf#Y3EhM(fkYA%6;q(E^Bkw{8#>s%
z$aMAU1GHyO)vR<;xbgD!>jY^3_k8ur7j|34I=zN>+lfQNOFB<=yoE;esh^pbmCrLW
zGGe7R25zXi!@Dvhm-gVQKb|~Aii*NByA-T4$81V2hPcY|J~<^HD|WoJ?`Fu3Adgdq
zZj)OC*q+=1rBT>9T1~$GElP+(Nl6KNsZ%7p_bEciwYPUTs?w)x-E5{ZiAhr=skMD=&hl(Sx2=lw>9Gk@3WZxf`B31XZN3s-C-EXQARs?1
z8wB>2W+7wQWtu$w?T&-c1_7hoJ`3w|9av`zY)Z(5J^Jf1@E)&)7CD&?v*N#XIU1{#
z*~d_BO@e)#L=qH@450@BJydCfCk4ykz(DMEo!XyW8-Fh3cXbPTKv|esDLp*Z8)EZJ
z`L?MU*vMekP>@4Li(wzCz^Ny_z61j|0v?H8D1~-w
zJ^G0I-__{QOIA?f3T>rmEq8T=%m7bBd$mx38of>>(th}Cno6#Cwq9c!Y~!Bv;fDri
zcw<2_KejK8=AuT(*Ph;9G+(*C5+ELSBLKn-oONQQTSz;mO<3aEJY^~S?iIF&5z*BZ`?
zXw!>0ys_##YFq~=b_@eYn}8kT=dNqN0$0m`H*Aa;nq^2-|f
zk-KrM#c|%Y=atlh+&g>=B5i^8~%k|WFbQ(^eb6NJ}{
z6n(!LZK87IfJ!Soo1|9xSsrAmYsJY=?|bnF6#VO5wtG2)gh)D#Q2;*
z)5zeB7f`F=vhPAdKZBRZT9J_m)OL8FWOw6MioXaIGv)W>+l$-<22863{1e|-&yVw2
zghj5ocr8V~zRG_I=EIj6v2)si*FHh0nnNAD@f@TF!=ACdrR6JbbYp3Z5=oB@+H7{a
zEx;VMS}l4S_MAV~k}v(ZP$eQ~?k7d=h3wmVbSDWv166fJ5nUx>?Qhqqp&L@$s)i-M
z#rfIWN{5u~g$S^_b@6=*#yyvG8TY!e)-^UO%y4nUD!dp^Qm@>AqKSy(x8&cHMqExm
zT%cIPi4BnUHR{IK@?*eN<4o0CZFJD$Uav8F$E5&0%@aB4VpWP-AlGxu;^Mc{S@yp*u^x`gZgw_YdELqEBC)L_DnpcddP
zR}vPrCexR63Ua}|#=8T>O7YNcc~;U~P*ZA5U0u#WNzPo51}yJ;Lf!F{Exiq}oSQgaTZs=%By5<0M3Ij+^g6`0`6gQRwqflub7_WdOCXTMGJSb4M#X
z*OIOFQB+SyAMwj{u$tE3rpnbQrzxRjfW!|c?0_X!_4)QCIn&LguAyQvX+^0HYl0k4
z6tVJ>b8F#q@e!Q@E}XWtwDjrRq!7*Wazz;+t!zRvlvIUJWjg)?JW1vKP
z1Y48(#S!DNd|rQ3p}BPVxZtfD4$2j$seI{5sR;y7)FqD|#65jz`b}{$N$#GJrrG<%dgOlj2enp7;5zLl3YiwD)u^CgDc@Hle!tV@{puNvGdw!|`#eK3Ik#no3pRk=mof*2PR
zF#ts=<(cgMKn-f`oc
z@9b~ywbz<+uDK451$RuEx3msz$hhcYpWhp8?JBRmEb?i^BCm#fgLwY?#J|A4q$}>{zP!xz
z;wqG}V{0bFhnZQr-S{ucu=x3H^U82P%3Q_6dJt4r6?D<}{lMt(-7K@xhro5rUSgq}
z$+f>DXj#~$U1j+9)flkmKRb=lpw&gCYr>3D%)^X0UN%1RsRL8Lf<)+-QK(Ic(E&_JdJ`X9(&sV-c+
z7zZ!*U2Oh|#^_!RDrmssjmi!#y00buZOLAtG@|nT^FeiHecHl84(qBJ7#PGoc>V^Y
zI~mEIr5$Ko9`JA9xP=Qd~|b&Y#1ZbZHn+nYlYGb(Ua=k
zgN&)P-Z(Snvxma)h$smHwxGb!(4k8dZYlNk4-uC(#329_w9wCP54KGBWc(_S$Ekq<
zL{}t47IEhGWtgqgR*v3GT`PtG%uvYPfwA7kSOgJ01)e9zA-5
zD69beqX$td2B|!t2{0z%DnYKfhaPebBeA0=?!xTaR9emN>n_%ZEJB+HT1WuvwGJ?9
zhD!qYJhs~$3DmGqIgqU(UMJv_!~`}9gs;GS3NhG0x`AEh6CDSj`Tr&+j`n|78c6>d
zrLC0AyHKWRDXs|AD?Sf#`Lj>rpl{bTQPD2sWO#1#u>P~*?x_sCkzJ^da=?)Sy7Y#?
z&_o5yxgfWd2HEQ%taN9nlf351NmVjX#D^yRXxX1mO2Zb7(iqU9HVN2922J8KgM7wVgIWh8*E7EG_hy)H>Z&f>K89wbo~1?t_rYH
zamCXmwNE{@p-_;I`i9<
zOMuynDFqtvU7(M97n}y#z)JyIDWS@VKs*+}fv6icenUdOoT;X8vX)?!Is!B9+c%fO
zrq?Q&&m&&t%Y}2gSm7eBQl-HXxB<@vDyer&@P`b*!o!eQs(yTef3oqN0P!;g>F|9t
z;--k$qk=hS1i+;9>B_hRb8|PVbLnS0tl4!a^si^_>wswa*CP>RGV$5)QxXg+;J0N|%
zAZ|wdMi-z6M`Y9^in3opk-Fh!p%Y2N9mzt5u_2Ct#x>{Jz6^c!0nbA@NSl-$74$v-EjY+Xmz>d`9~=C^x&)*@9Y1%i6c
zEr$Hn&iz@?sUynFj+Zx?qM)^>W{|_o%rB;rbK&JQPy4=SjE`@{#fG97c^nqo)^9Aj
z)md*YT$aK>m3|Mp>Y=^7xmJXd@*#M`A|h@|4&Gu}{wiggKM|GP>cQ$BH5~KuofFPa
zFaw}}2r>V2+sp&&fjIPdBMz(}Q(Xo#M(7rA-CG~8=ub0v5R{ymtLv6pj-0s)f$oJW
z`(}h^bgx;;qRA5|2vx<)l3%^g@V>o$B@13A0(cr&pJ^Y5`C?&X5#&p{XD`dNc1jrQ
zTwXlDjo+D%{Ri&BviL}~^-Pz5u)!cWXTqX}oP|A`hZI|R@N$myASLr8
zSb+|c_a|{&kE`U90i9H~$w;-!7f^2z>~IJ$z&D=az-t*tOl%foIjNDC4EC2|P}4|Z
zpwq$7h!cz|3~wu$r_Cho96f5hYu3KWxFlla%`zDrk(?RSWS5^d+NimX<5dApfq8nK
zCm@6#WEa3=r-Rx-wtaRm!Vsrqwn0@^!&N4iK(7x|Ghbq`&Ns+}QATbYkT@wn1f;OV
z2Y&c-I1YoL>RtT<&T-acA|8U`ccB*0JgX?NErJptm{gq8aGSoSZo%t@xu4YhZg1HH
zBjbZh%kEeY^uT=(tp8p)TroHY)<;}{Gb$nX`Ro_m(rR@awd+=db7h+gU-oY@B26sD
z%fyQ9TL&{Q?AZlaU=hf#0psQP^}a0DyFjvtG;vmG%vAERjAe2`2A=?C=;$Rx4`B;E
zxugPZJ!}I)%^HznyG12;(9|RC{O(|GjLNO^G!k$X(z12>2`T4(0&r>=#5&9SvveUk?lP14
zA|Li5@vw7L(8)sds
z;WXUQQxF8)-Ly)Mdt#>=QxPbAWEQ4O{v8V7b#Oz%p
z*nn#b7#mEuU#3JbRyaJF5Py-z_M5bF1BfvN=$jD}W`dm<}pvv_B}m?(rqrdo|*hfwZ|pFUChid(FtMp{U4_UDk>^)363$Gtm{~CdmKz-
zA^2f2{<|F&w~UBrI>Jl4BLgI#bOXiks+1lMK`BAEvO{w3TCfq+u?
z0sYZZ1j31IvJ$SRI6|dW!65R8Y>AMabSO-HR8idl(kB9QSZ^bVcPQCpVI8nj7>2%_
zbuGZ1!QaCaoX8equ7qiM6;@s3Va^9dvPo0agOmMc+{Ccp-e1nHT=v`a_?AR@48FH7
zz&-y?7kD8A!4^5){|H`^`Owag1y?9ystKmG9ic`H3Z~g4lP?DZ%MG(!TMNeyMqI9gM{uIkM1wQSk{7{aRcJM@X6>E
z9K(uZp7@j$ypy-&VotBF)i%F~W#xvLHk@L@eciWOX`yil!=hK4mzm@u1;U46{V*XT
z2PP~p%`0NP+lTGbEtY|7+5@y6?_A!hsMZ~dE6Y&eV37<$X_5_XN{HQ}C%CBzjb+$R
zIwC_NK&!zCwqi%bA}GW^9z2ju;&2z+J4bSm2)c}Sursn%$7%aChImalclL-i%hNk`
zNjcMhTqo~uHDIL?jMa+745YP-m|3f3aB#O^6>Ow}>|`RX6+T>iH>e$R?4;f!
z#ei3y!Jm%bIaYMpzfQ7`+LwCfWr>Vx2q^p815aYXdlvE7_;5?~Ct<8mNElOs`b9oE8dW(voe3Gij5_KA7kI!RGbJ
z8%_SKl@I1A<3hJcyRQ^29Yw1?ZlrlR9;bikaAMs^{lmb%C<}@qmvlwk1DA40cH!gS
zz{k%SIKUP4EEG1~?!ce;-QHyKL%I(^4ulH@4ksc2qEYrw-(fYlsB6)6hfqn0Q7=D}
z#&C-cYHP^x27eWE#nn>F1aJ#C!>61PLGr%zF-*7*Z1%~sR>7baas1_!JC7wPUnQ@n
z-rjq+1nEkys>oVWUeC2jX0x$*acT`*sxkn;?qeSVGxiD!rcOr?jf`dYWUiV%_sg-?
zT%%?rOBpC{;0QuoZNQNBuOS(*b3mm9f^91^8gV9jjdKa?Sue^a@~gt4m$J_NgVl13
z2pQ=wN4@knEU0bW&3WQG1j=TZ8*>3GY~T+-*ybZ^C!S-VEO{?y;&qg(BN*-BUJ;sM
z&Y#Yzewuz_Of_hIOatNxdGGj0C0#v&zj&Mhe;zi*7codj+=mc{LcpTIpAW$P5eYvq
z`%*;alYqA)j5M*Z8Qp1yq@PsQEmf9+NjPPa$Fg5=`1w|uwz_tsuFb2H$dO*?QG<7A
zZDXVK`rHMu9$X&K%9KOdSO=*3kq%lO@MF4q{W6IDcTzBLxd
zGi@BKcfJ4CwhsZ%8NjOUa3c*vyBeL8^%PO
zk$)b`!0q4aqFx87oh3g1k?~22FVYFWuFf^KDi--U;T_=7KN(T$SjE%Wl46>rBKm+}
zK=Oh|Y4wla^Mz<3W3H`XF@6dXDs+wxQ?%a~rjJw3)v^%6>udh{v>=@am$)69NlGc|
z6xHwF=W1NeO`CeFZjpU-4lYSj;$f$+;efxF7QEBTy*St(gdf;CID)UoKyi`#P?gP!)CU7bSz&pC~P_v8XMd(xn(;$+VsYX@P;48kwD*^
z`>ltU*zd}#8-vU(JF;*{S<;FeN*~!#CnbDgy^oM-zJK{~18{vSTF{)Wy@A|_J(rP3
ze-P@pSHd2um>tkuyQG`vB^cR9Xn9f*73kYM!dmqGTP(eG9kH4z@?#w>q`GA(g?Y1m
z@Dke5Rhi#^pD_S*HspUFeUjD7VLi6W4s;;vuhz~#Otrn>d;aW9MIF}~Q=D6Yjh)8$
zujpuvR=76<3b4g?B2RjD8c__QQpdMYlX7+!l0gK)CnPap^q*3|;U4a;r$%sUk0mZU
z_~xFJhv3zb=YOlHE~)&<=4&8u>WBl2i^WTLD>uSrkRfl-dnwnfloDi!5V4j5x`#}6
z)+In+`|>XksH}+f8fljXqoypdt3&8@#G&ZyTi9K>g-FdRz;cznF%_k0fQ=P;K8@>B
ztF-BWy#`+9vmJ98;j3*Xy+Ng_r3wz>&9qC~M4s8bx6F3OYPVk?wC+3T6M7Ay9_d|0
z#vcF-hPa9EKuh{EY*Hf$94tYv!MChSU<83evktK4P>|QLbn9J0t>$e7zyIhTu!vRf
zc8gVa0AhTp$B7Hl>$k%DFpfTcQXN4H3#-&ryuzxww1bAWl^qurDZU`W>W)z4yVvNv
z9)j60GM$N-MLP>Iazci5aKDOs^X3iW+RO~bzTEMhD>2Y>Oj=Y0pE!%B
z?Qx5V96@UoC8#gC4it(sIzn-=GQj)KuhMb703JOTl!wSL7Gzcw*wku4w;IH{J|D=c
za<>jjOcjLiol;}86BBoudl^mnO>2;Y2<-@eG17My(Bl}3m}0}7f-rV)b(TOUBf=$u
zm<@bTLq;~?B_LukK;VZ%ew4&O0f(prB&_p8ELczH@mJCr-yM{cJ@PNvQkTbUhg6XAOYNB5!^{C-rd7552GGX
zkH!#RhW#RZ%5V2qN|hd&f1VTt?ns601!ez$?Ow6G$FJ@d}1nh-V(S
zF{*L}ni35iD+{bgrBFT_+S(2wXm2#*L%ckOSr%&h
zcomYcz$7a3BJE!dnZgFcc$L6h*rw(2j_wm
z<#$r0+^y(pXG4OY$?;8M6dui^`2q*i8?mH6^JxAg^~Qa_bLxi6hsLB8HIPtF!3YhM
zuy7I#q~Tpqks=Zpz(_^p3v4Z%3=HRumv|S*=4`usEDK25y#KiIn-|aTl$&ql6qV==
zgp|KHpV(2Zb}-J2bVqNF-)e7ZWxY@MX1g+DV?N^Q2ZzD^l>P`*q~Z_1i}jyzMra%^>5zW>&nFVhTsRosnR)0lV+D%<=s_IyY&|NHB0KpkA|g%69A3iC%1A_78VdYOTfyj64mz?)gj_hG8G0^K
zT^E;MJbM6Lpc`KW0$bSN1`X
zRROv=PX|FhzSy0e9V~WCTU1uowRt=A@8bB-*&Ka>`z-u{tkLE(GqklcwSR~|*-Pm5
z&`V)ZC-?7Cre7d*5
z^gH-glG(XnRDK(*wagbqXGTOS*ORo42->{}Ngf)t=((Dj>Mg5)Y4e8>Y7xkZL@VH!!|Y44K)~CAi(86Iut}=f|62o(8}VVH*fJ9FqoE(4ip%NFW8|}
zuU+sJ*B0D)TB}tV9*Q2RCUT_c{cufthh@OxIy`$@OrUPJfss)k2zN5%le=Il7)*J*
zx45_n1H-(*yZ#-3OXm}(SaTMJ<UD$kgZlSlLx!1fsc5Q_->gnR)S=Sd$zv1cvxz~P=A^Ms^#Ia#&t;`P+B0$(lW
z4@$c!$`NVowDj}=;1anFV?#j9Mhf?^92~3#&@%R9qYZUs>s`#XyLNH#*526SGWCLcIJ~viWHL`c4@|&J9E$|4)VMRs
zlk4Q%#=sE%U83|%?0AS;A^)B>@5P*ftNcP}23uX{St1)-TfVIps)K2YvliyE@kdb8
zzj5)xM|_@K~uq0-L+
zph!ySmlFs950ZXx0m6d8EZ;!lwGZSDADHvvbUnB&*BfC+DXUuiHf3dGPjc-
z&C`sBkL=5L^s|n(HC-?}PgQnYX!!Z#N=i5(gRN@c1Qt*c$RIF&!D(A5%ZB%2*l!GZ
zT>TGUrpXxm`sesJE9Q5G?96l-e|4bHqMwo!dlInkhYIa!)SjKXg39wd1i_I?dq&W%
z8bYE$^yT0l`chw8$cVO(8;!WQZ*bm`l=U{%7xpW+4IyamiSm-t5aOMzIGLIIHJsWhfK9BA_`fv@g@L^*UIT?k%
zg%*&)@gB@X;B+W3OisN>Eniuf+gXh15NY=8UhY@h{#wo12+72A(?AONMPxa5DE$Aa{hj@@;zfj=IpP@q#;8h-Lq
zPy3@n-{~Gi&IdcoktM+sEp-J;vhfDd`r(9{L^Vmyz%uf|2spH)dIIgSs;;A%Rra-!_3tY;=^ZRTT~^G4s3{XKXI)))OH
zWn>Hh_#w6H5f#l;%K7mjs-(`X*zG3HnLuPg$9(C&g#Xofu}o2`rkkJQJ6)(3|Kw!t
zb^u8Li1@t7*#Q9-v`LYA?EhxcE`JAR_{s_j^#~{cIaYn_X;Gldi3%GbkuS-a&c$vS
zt)>;Or`Y~@Urko;*bIK|RI2|M8
z_-s3@xbt0rXjM7KcK7ooYo%qE$KnUiucH~vATpQ4q}){405cmb*<)e&{av#fCU6v5OSH9(NIN}-Jo2?RF3e}~mh7dPJJWwf}1uSvia
z|GmEK-dFj0Nu@M0mxT}2XQxM4Od6lOkc$*XMnos*m70QRV(>D95V|haI=@Y@rwU(h
z!sXmBspqx98!t3!@5Mj;nbgh}=GJsskIC58LbS(33X{fOPeOU0l{L(^TPQk&vwD
zM5LO}9iP>$;+)ssiKTbKm!2⋙I<9ShUiCT|0A`)lcX7O%Snvg1M}6Lc1yu&(&;B
zVIt%VM0J7(V(2>YMAOaj@mwNxGS_QVnfY^ddIgk1jBo4Q_sPy;%Y8&8IA^D*;*@et
z-FmX&36$C#s~cfBf)k7_MmB0&x(OqBX^_?QnR|j
zybzdT7J3(?wZ^3ufOjER)old_%${1Qk46FS9A$0wuG8tyLgr30^?Qt&US~4eZQ-8D
zEkf&E#+`FyqM7t)jMP22r9BV(lbyf<>*Zvv5abU>BQy(Pa7#Rlz0fWjhg!|$ig)Ze
zYGtjUIuj|$jcXMhu8~)nT@uZt&OY}!iGHlwqi>Cf*q5a4L1HE?v%ZBX4OXw>Jvqx^
zuiQ`}f3ds!>*5%x%gZDeI;lT3fPeA=DBZDOVBO^G_?d-23;J;N%RI5wNPB<9{dL2`
zMmIDsZJfd=6Ys*l-kA{^4w9msihslyXUg#N-Pj`Yo84};!APDsF|zFBxW^wVl6F;{
zV)#vgtl7RmJ_=2gfPZZhYVZq*Z%~npkkMH$<5ZSW}z+SlURev`pc6I$`l^Uo%&N&!5$ZpFt&6rrf-F=07kKFuIV`ZR2YKWaDY+>DUjz{ps|0
zUgT0jA}W&EvQyGJNh@0{-xj)X3O#3}ucG!z=c;Ga>NyYVdfi<|xOrndXIs7g@S
z^VY-FaZb~#%|eGHtRAIrR>t3?^n-i+C4c;D*095F+`Zcl!i;RhYibkO^e{&ILQE`t
z3y0dPx!PWieX|wm$8W#&Jg3|_mhLe@EBUHH00y9@4gTw9NDi?y$}Gshh@XI$1AU}C
zVTwEMV#8Qldve}FAf3oQr?HrYHv`(k2h+6fcey(nbY1%<4oUSf;*|^PR@Y{a|O)hTmI@*vyrB1uV7Rp4QvV9Ma9nY
zoh&0Jxyhyl&FpFcGqOcZKBP?5Ju>2wHB-&7%zOd~os3jxa$y1`#HOskehntBt)@F>HaXP|^NL@Nm$ShTA$;Dj>~$>laHR1YfTs}zz)FujBP}ZHhjAq{Kg39@hFaE55}XlBl7N#oxX$P
zdrMDkKIUHY9Fgcrafk2jV_|R+iinGog1I#Ry#3CkS>yw5U!M$GZ-6YI^Ol~QC|u;;
za+|kQca~`{FbkAc%-l}pQ-G8by%C@~
zBUst>Fy7=Oz6l1hIf)o6@jH7*mC!DKS$Q^Ecd1W6@b$3*Q}3-%nw$eRh}vioeIXoZ
zF|G#GtOJHb8DwmGVdG>$!*Ngc2}5DBic6Z|+DxJ*Y!;44>gBR%(n1}sP>wO$e)#UU
zTrzfb*U*QaNQHNol1RhO&aM?Mo;$~upNX%r1turNdKkf8!#viKD{|(Z=}nAKU3r
zS~^tVvB!7Gm#%8aQdp=$EkjHD*2l*urGI~d;cz!X>Z28J#~XXiYWU7SMuqyy4!W%s
z-pi`#$fj6(Cv%ckOkl6|c740s$f
zA$&4F#$(kYLsQ3a-Lp)fa*@i1Neg{o@Cd+RO8{+>q{{pD0m)IG4+{POY?Ywprb3ko
zX4|RnI=RkiHL-U5ItpGDJnp@JDr|9GqBKi3Ika_$KTlY7>O|)$CM|^*C+(cHy!<>s
zHbyW5&U~UW5*Rpk2-jDIc6w5j8HrXDqz}(d4_aR!#ej9MVsQD2RV8OG5gd@6AlL^S
zFn0xJK0yZI_E1Ub1~N7Uc>Xkyx{7A*#jC&$h@_4c*gmk*y`yhbU^v>=N!`rYEN*KP
zf4H+e;J?;scAaeYh9-uA7`aw2`eW^bQuH>z9UTzCsUViQLM|8su6ODS@~yBHcIq;M
z?|yaEt7*r%faAQg61uZKPPgzLwWzMyO$bBQ?m|Z#G4$;lOf>{>TK-IcA7j%k^yhry
zDC^cC+OU5~R^T>=8S_Wu^diP|Wo1&6^7oY_SkYO*9L&yhx<_UhDtehP^DH)8K^x
zR|jyoz?>5YV6`j!>Ge+V3?J7sJ*C=?=9TH+H!>S|ZGQRf_YUov$X*U*XU{JFA{#uZu2NaKRLe|*u9uVVMpdVAI+QSbtnl;m
zr#!0ls$vA4*`swFNMJ=Dz`Zn|otcyhvmlV|{(7w3oOric_i?%cxh*4E7sbVKQY2E<
zypM?Zj!Xnw|5>v*kbM)tin+P?nq=wGI*7lC@dLjs-AA`pb%dOa9@4syYlX#RUE3Zq
z+JaK8Z}ltg74ki1iVj&S+B!QPLuxcb(0D-ZIcaHW1R{8jYzrtHKwh~4Kn7gy@krvP
zbkF#^)T6>(s&I4S$7QJ_gCG4vLoV|(qx&ooPKN`AVnbp{25Q=c$vP1jCWPBeYAXY{
zN6xb;{Zu{(<8mfrUb17hBmz7kur#p|ajib$%dwwc3PEoovIA>k#Zs@cF602HLYxRDJwxRpqzgY=r~3*J2B^tQ9~t&Nn&>{=+=bl^NHbuR%!&o4HUwjpa47c=5`~-xp@$5g0FZx=%v4N(VLIzJE6vlc)9~vN?Kg^s
z1``t_3Db}EVKkCd*Zj{w(bZb~sV9Q14`<
zq&$megJLNa>bZvI=6)C>1tR3VVr|4O6__z8sHGr^>w^^7b+=eDr(8HD`Kx>UX?>-Y
z>>TMIOeX=FL9D2#NQ(UJ_vi4}y?ttQy10spKh@QoPLVE)?wzXB`kV9izhbgbk)_~}V0zcI)*pxU5fT@^dCO!cTKagqcCMG7@cD>>a
zO-)-c!|hySNIZgCsVS$hAAVo5`-=2+$D!An}u
z!;wfGX%RV<%9ks~Zfq}U&OahSCtMI;P}%SiR_RXWB24g&yJ5Jb-JB+=mqqodlKoiOEcI<5uux*TGbo`8Juzox6yHQjpAk=0XjW$#l
z&odT2JGKoK+>(T&y$`X`(;9dRa#ydSLwAO$-G~z7%fp6jE=CONU7F2)uPUgU!z?u8
zu_xkswtFfY9#{3eP3vn!@SKbOd-31CJ%ZF(+`)lc$YtjlsF-Hv=lh|j62k_`XevA|
zHU2vw$9mxSVCF6%Z>t=fR5U+5!v-+I04Bg!HSAk=;PZ@QJ%rQ_ZlYg+&9m}R5D|$3
z#t8%mGK>E}F$r&Ys*S&UV|`m-6!c{53!d?JfU%&W
zrk*t}e)#S80~kQg2f?aO*ks~C_z?b6+cEn5m=Y(P@0kTJ`u^PAw`qM+t6dcC60n&t
zT*@|2mh{LwO-eMA5s0Q=5;%k~N2e1R-U%nmc2HFv8SMsBm}OmD_@Qwfyn)@LH(|d}
z2?>#T0fC?sM^R)6$L$lJ%5E
zZYCX)vgrxo)%Rr83fgZfqG-y$rlBJ3;TPz`c@qVN^&t)
zz64ea;Up1q$7`jzP2u5@?Kf{mS=yrN?F)&bjM+~F#c?3(bfy}1mUPqEp
ze716?4d^I2UiMG|bAH=St8NLFn3jX1c&1(cf)r6hb=j|vPqFvazm6pEZaBmZ*pL~b
z83;Vam#5FhAiu@M#RY5?9UiGv%wCo-9@tuz*izTME?H*8MUVG=-s%|ib33#?wK
z0q+2{(i;0^6cv>>u--TdFzOO6sxQ*j!NI{8TanX0&H85oG;D0)(4c#rj7-+fZpWhe
z__n?y6iESLVUxMZs!AkfcIU)D7Tynh@EuPs_2;5v|8$C$s4+EqR05o~u_+1htf3zm
zZjo7xEp#}`B=SeXAiwkP6H(&2P!bz|v22N5y<3FRzYDu$ZZ$SL?`O65TM?}W{baXp
z`d7;wpImQNa%O)XoBIU6F`VOJ19<0l!dc%bou#@LJ{CXe<61mv;7r-2Mzdvtua6@L_qvU27)#nTLx3WFQQ^$35EZ6ojn6+Cfu7e
z?}NipLtSYbyRRSnZaWu#x|Cb_^ZSK~`16U8DEf2%(La#Hcfe@QnFFALkU#^jMaD4D
z7AfhE#M@m4%hx-;$G9{H2=*0{t=^#ySJ!B$Dh4dJSef@=lhxUHbM9{rpGJ&X%q5ODW%LP@TfOc*JlCi3q2QMyBS&q*aNQB~7W86sY2$1GJU=e63
zsTVLy?+OXK5A0sJ*;?#}b)aWRXyIp#02Xbc-3@rlbI^)06D~o1Pe>@SYOONJte_&p
zz@bT1U8fPLtAOA5mL9F!2J@?&L7|HTL8&byyu*ynR7gq@AXe~nf7-y<*tV5A`Y~Ak
zD`}S9tK^Qqyd18m5y>XIKIhv{aG9SPt>km8@m<%Zm0A!6Pr_AuF(Dj9<`g;of){wy
zc{|zX7E5Y=p2cyF^Ld8WP26kFj@SshTFlRB@UV;qQgJ+_D&Gr&($7JHi48KMf|@-o
zFh%c%RKKNay|x{+(oK*!SF2mY)VdCM*OJoGRB*L{7Q`BY`PN&JV{9(~uo38l114w#
zk%OT3FoJN0MH(u;(m*hbj{^e?#dIkmBr5{gPDZ9f{-hHA8G?=!K{g7?hz6kbI)Nsw
zmSzFS0tJd>I-tW~-s)@5v!hA7<)KxcGbT+7(xU_1%Fi2_)s{$QRD@aa1O1W3t`U
zk@7bgR1_vGU;FR9`4N~33AL)tcXLxX4_?5^0UCv*vZfJ;>nX0%hoowf2AClUs1h*XR{
z-yiWAlFL*&eU|#sWvjzW4nXv5Qda;7Di%_M7GTYh3>MjeFp*P;goI>gv7Z(gA^HbU
zcUg3r_2xS>b6Z$;W-Vq1$9xF{l#v}6kSixH)D!W;2G9P`03$1G&1P|r5zP>9?xmst
zJ%6LCo1lDo7a6H)BTR@#)>48pIiqXw=3#wP+COrmpsgfTtb9H1J0N?jDI5`~RS6i@
zNqKfSTvt99X9n5i9eKgc^APfeIExWila1NTrRuDK7rt&?e@LSm4{{m6j^+FB7gd
zXPU3xbje`$ssL?(s@fMk=6V2?fvc?%4(-daFU+9eA)`&OItMvd9p4ZX|6d_ix4}qe
z*vtAbmv(1u!ZmEVzprn9JNc{w)=jh0j{Yx93o!Q|hO1uYIv4_D`<*p+e;>y^0>sl==#b5!UZJ9h#x;+wI6_S5Y!{YAPgAPZEWl6$^d#$aCbnt&cw~af(3TN?}Dne
zRQQLk(dDQMdCzyHuJA2htF2u4UfmpqSFF@#ps7V2m7~@S7T8K
z@v>Z4mjtpsc8_BxfTS8~0`1%AGp*py<2i%_MeZ}KUl6@UbB8f$efPQf1C>X)e*x^J
zQTtqgDx<(?AXsI{VxsjU1?!yNbX}%;{Na^_p&0T`vx?v^sO$kyZ}7Z*uVUeQ*%lZ3
zYcUTDd+6N7iL@q`tSDG{(+L5J<}{Q*^^lqHzU*!()q88@8$&C5b|VKO_M
zs|>y#pufLgFGS%kJZ+Q9&SUrB`$fAc;McstaorBF+9;l}R#71=n{asQHF8`D39UcE
zm>>e4I=Xw@aUO|xdkbx;hzr_Zz*kj6P-y2dKxkl)Z(KJnv23YdlYKc!*Q$9BllHLU
zqCcW}$xMX7TaZ>kLSczDmt^W#00)j5apP#XIoIW|I1!+S;}A
ziyG6Q`LUpC@z_>3u?1)Ph}c-otx~uR5#~^Dpz`^B&*lE8Y{$Ks!kV$391kv=%Yg{a
zU6Xtm*MGK61T+YZKpEwN3N@POJ;%e6vCew{J{(nXzkQ$fFhBmncR6F@5Vi5hJ&A9>
z&$GZQ0?i99KL@%GTM2CY^-fBx|MJfOE;>Qt{OaE%gn%C(J!+;pt$+JR4K7Y4m(ko(
z{*^N$BK`UF&(Cy-%fgbHy(!!zlI2?ArTd62sUxVb(uEf|kFQUVDq+qQThmt$QEYuF
zekR5ezn!|+TTSfAxBVbYLkSvKZ!r=wlYtjLCb9RBNf
zS53n(D&eFMQje(DV1x=Ba+B6=s9Ybw$O)LpjE3Q%72sv0S>?bCO+!+xF`$Boc3X#<
zGe*dj8(v?D#e_@U3>`Wh9l~I(?IB}(kzz0yG)HW1OANxAF<;9V%QjZqn8!~nt_Tt2
zH@Ci8vR(OVFX>-OavvR7{U5xla>vlnF#P@d$3{jp0RaJ`nN%k74(5IzGQat)|F|73
zQ(h2V-}hsu9G<|o%18l_!r>aGZphm`UhLH!FQnVZN>nxNP!HbNl}7qH-1_s
zjxzvyV^58nFToZ2>G}6^Xo}FRu%U+XnSrmYio_o}Z=VBeRkz4hwt3T`0-<_`E0E27_i*UwjzjkqEK$>JWZrygv%1L~Q{i%%qD$I%
z{FrmdaBc(6v~W!P6E_*cW>W39$n!RcEr+Ynb1#%oqYY!0-}oa65U9r3VQ@NJ#+~q{
zv12%luR^1+W&H@ckM&389d2f75$^EQG0Xd0+HVEq%{rlk#D~4FM87dycNz&fPt+}4
zr`JDzB>#LU0JK0XpRJJ>u#t_#L`135{c+>APt1@I{Ih0GvVeb&GHxJ8#zOS^^ev+Y
zGo5^P2g^<@m&K(H%-lP5l}05uD_As6}JI
z&46<57f$CdOLF^XGX0f4EE0m9V|K@4p%RP$dM%V~d+}#A4Q-jpLm<)>%d{T#WJ_b(
zuz&=EaV5dG*`|z}Avte{R2N}lJVbe~#AYi9`F4&tYw!FdR-5O}d|Dhg?x_|(QBcN$
z&WTwXF6;(k&ICr?vNYF$i4gP_k;{O4rVyU*qY?{9_5A{nD+&OsCP0o3UV89f?l0PR
zS){4etqgGenJ6Dk!^N8}iz8>Doa{Hqcpn#qi25C@sxc9%dHn3zQ>etdEr&ljeeOu=`}WyD)dr3SI!fo$clwL|?^
z{}I;yOmTpWj`ewnj!xDJ>Imh<4ReFCZO2}fDXOS6z|GkL9hJOuG5Y>tVe`l*z?F6x
zKA}(drBW`QI=SmL_VYX;-Tc1@k~^JA*FUwS{5dxbpVP^S2V
zX&t!+My0^_+ixr01r%Xkb+uqY#oQf;R!^QkzXv)rLSkZ@uNQ7TfH`QXA3u7_v0eMW
z=`$LDk)B1#Lkp&A=X*XuF~d7Ax-{%=e7IK4n{}DCBSTJBePLp3B_q-0CDGKH1?`52{r4j3Mds61%Hn
zTcJ&lg$PLTuSIGU8{q@ypbxW^>{fn+Lo~UVsFVUZc6v)&;jh73g&1kA?NibN9g4tj
z_tpe=6<9=?xW=hX4w})EnEl;wdbF3icr~obWZRl5qi(uTQof(qGc$kA@
z1J)Ba$jCZjEY5AvC_07`9#wMZf7LrG*}O#rEYAdx#eyJRLR7AhGueU=1}Psm0GrtOp&y3G
zlGJ|~>QZN?0ud>l92&CF|M~hz#RVCy>emGGnVTc2yLs&$ztaoVH~NMddDC>c4jnBT
zG|BENAwk1L8gsvc`V+$^cyBiSvOLzHU^uq{!bWi!ClJ?31$=YE
z6Y+e{7){!h$!LRUY8?xOFu)>sL!K^G6R6dV@E*{nn}A3(!CRvrd=pt}_h0C`Z8owG
z3KnJ!%VYL36OJF%?t9>@cWO4%Ugl_Fq&r8+&(OfeY}OC?7s3$uXxyDuLTPl0VxjyyY{$1c!$(l(`rT+cDn=-3RHC})%hq)q9>>J
zjD~u#EDNMR+S;UyYI&4=d?a3g!LYp=-`F{e-oHzJX
zI+T{C!RpD8-_G*5e<=N6yUMmU!z+ZSi>}Y92cuz0$shJE#^&ZsUpmkl*8FL&_jT8j5&oy=#5xLgEKh
zBt21S(!rVi+M200-HE-0IHV4QY5&r+Qve>haf*Z_98XafJ-Wxcl`;xEm%r}Y!AzwD
zX#;fK-{b+F#T+zlA_Cl2zv{8r!%>u^exHYL@ra~OYdW?`Yae#e{HGvRZRRH67egzN
zY@RJ^n`r|_Lj-hsR#Yipj^Qivg$X~vq&J6xRWkB_bf`=7Yz_8>sDy<59{94mIGla`
zEl=(u5OgYmdemFZtE&eI`s{q2NR3f_l=?jb1TvU;T(XVEj9e==o`IoVTw*b}Fbs}IabIOtJ
zIM$0vdp7v;+y3k2qiU!<#_g1Yqa~@^Zn<=^~{hh2-wi*
zu}++?M}+c*5><;8s8z=QR>Dx8m(*w|F27ocDiW0aRE+bP)J2E}0$B;*(J%~+7h}@x
ze*Jrt(}du_u1b(x{xxBia;88$KR5?Eo?@5ns|?NJ+dufoKpA$vs2@pkE?ek?Peu;}w(!u|ngO?1mLBCEp1MY)N&YLi!l-!Ub+`Ra9D!wN(jqoC
zXXMk()Dm>(CuR+|mpm~Aq2emVzqj1q_jvSm4}U|6^+$DZNF>3OPiGlMTs4$8i&$Q#
zC*Y3i)SE6)rxX3RG_%Oke63_cf(ihZMTQa{trcHd(L2lUJ7UQ&*^BE>7-;`x|NF>+
z*}cz1ds$bQtZ(iUeFk}^MP2+AQ=8Us*^2D}vBW)Q^wH~GnUT5Oh_o%{!W92*;==h=
zl5836lB72jLoIk+G$DGjiPBP{bFlDieurSc<9?g}0d;6R
z`7LT;RIo1-A6f=*`2~<|!v70&zzg;o?k;3%$pC~qTXymPgot!ps_9lu=^b{QGf
zPaCAY+BOCpEPO>IL4Czeu>bUp^7W_pwEaA4Fdy^^(^OBO7jPV~OO$e^mK!K}lwyC~
z(C8xvRy8mflRje!+6=*V|FTMNs9impditxvf
zlW5s3Lruk4HscgO^SEmf1A6?sN;Njxpal^5{QqLp;fb2(DSL>>(M>Gx@?>e&axZ;?
zf}2)q7?sWcq8&&!nDDk6G5Eo8wSJ&TYGGya2x4WS35?Jho=#w)Cml)U1|;hr*{{gQ
zYM+zF5GqE9cz72?So&NG4iHE@Vxf>JMq+?8^7Gpwk;S7>ik>FOZl7Bb{Fzu21x3&I
z)SXwsiSbQ8Doi+Kq*8!E+*wa}G_4hhgpRAc{}Va{KrE>1y3E|AYlR;vR^g^R>vXkh
z4?jmWkbs{Ho%88m5Eo@)>HGaTabEk2fgP@D4p%Y|YDjn4Tk1yGR;F1=#a~njwAFh;
zcKd%FzdD_{P&aayT`~(DM;xw4W18UuAx*=z@^*O;?(i`Gv$#Q!Um)s}&P6jMihHM1rVJ25gnz47^MmKJEeNa@nCrk{0gn~u%&TJd-CSW-qWn>qg_#VR$n>{zWf9g
zE-b}sw13E4@E$(jM|Uv<{aPT&Mzm;4o&Sft_m1bX|KEqTCmN)(8bp+tS*S#j>``VB
zGRoc=O+^wVBqJ$%6(K|^dt`*{J+k-acbww7uJ7;u-jB!q|9w5K&*!Sd`#fLcIgjIb
zp4=;NlgP5)h&4vNZcN+hD9ETlgi|=Z-&cKAe88~AM!pQ;fSYME4RRCf*rEhI?z85K
zcD_0jenjG%{rec$IPXKT5zNI&J^^+Xv-^c#`-)3D?HsOjI@E1x@O>oq&n{|OO4Ii&
z?EkS8lgDVt#>T^R=i8tExSwNcTlwmgP{Quy_N@m?Jti0;lM?>AlCT8>HECg;?)RLA
zv-|bpnpDj*tbg&(T1s{G<3Om#tD0_!QYWz8D;}^eGtllq!+bT3a8UX~Bc0rYC)J*jnKn`aF}#~9
zHasL^`ex4kT4rg3N|%o(`9dAWRDr#o3wEL=Ed8aQj1pM|Nt6L;v&s963rjm&C36H0
zpV|~sNJCoiyrnpUi=dV=UcQh(|kEsaSge)z>|6aC_tpl~(M?40t{$Tn(@+1S#W
zTgh*-Fx>V${FF8TR>ai?le>se01y4#_huV_RUj~l@7k@+Pc-*!
z49?YeFr1Csbs+Kus-F^du`u0f_R~^`q@E=D3-A>~4Pp!D
zZDgrbt711+-D4GYHJrG)ysO?MkKDGJDHhey{QAJ_e{$5$WKdB2l2%Sbl~nHW3;GRr
zE&nLHhii3<6PL22PmvTwM9N3pr^AF2L}T6FmOiFiUT{_QTC3aKkav0jP1RvLw
zDJLH~r~DK$p+K01U)&1bplSQKc_DBLA4^yyC>z{BBw8%A#I2J?r`LI?=B2Q!BFc(q
zy9^`jc2pTq5^NDL3AvaFdtZN|rGu9=KfmzWcyTh}4G{S&!V)r5Am4}Jo3NZNg>P58Pv_*e|FB^Y`E~sGeOcQ9+!(E5BFcXiw5gIv9-J%
zB~CrccP35&g|%sY`K7Gu53nd^(2T(!;wDeKuu@iLw9m}C-S>zxNGA5KT?
z$grS1HnU!v9BMvzgUv^jQg`&Qb<Lp~?!$cm8<;~#HAvGzxe9>N+Eayo4_1X>}
znraR&V!M@HA1cNJcV6gSxWhNwdouf|J&R}8`&*q|au!!!Cwjn5+5}oNQ_+_v*?vsB
zt8UvTe|diAYmr@KgOrw^y~7;;vT=cXSGp$W2ekcF2uQ64D5*Wn(tXIztC>r3QaAL!~fAybWrFmPjXZ)=D-H7DF{5!!KBX+nXwWlGYl9m)=3ff?BvH9uQYCY
zm;F9EY2Zn1|5N@VtT1tne#BDQe;(cHER6#p2HHIk5@dm!Ib&vHanY1)gb?*Cq4nzB(glh5AQSva7;(65VB^y
zh@#I)4R1viYvo7%mD>g(SyKXM~uRwK6C+bR8i
z?TylbI={t0h3TuB`(>Ozy%6iIx1Ai*MVoX3QIT~6Z#K>TU5J7Y1!x4Q+FbA6GdyMH
zZmV0!LPBubt~P(7alX)~vFUxu9j|WDYivn^w9?KEjDS<%BXlLft>CWb>lpt;>IWF1
zjMJwV0!E)kM_(ur$^4f<@3h&}Jtv|y7}2)b@Eyj^XSdez-}$j7WxBAXASBu9+!9Hq
ze7t4JgtY2XQA@JZvlP--G#eH)Eqj;Gth_%ZCFL@58CWJM%}HN>
zn;X_|RJXXjjqGO2E-87j_LG|4in+R-9b+wn4PEnHx=p_hb(sH@PusKAHK#S?
z`7_D4|G}o0AN6L@_%xW`4xZpXgdik9?bhum=Y5J~7+Q^g`d
zBGp5tIy5t7G{Ez3eyUVwTo+X2IKVtBD&gvG4g!u%iG39
zaziO=zvrZhOucy4wXxBHr8_c%EKaw(p8xFqw%`}Z5P4#y(!oG`8T8BE=uaE~?CEre
zB_2-+=}^zh{nc9^iFT{e>4`$KHF{S*{aZSSJ%EG(J6NREq;;XI^tN{BS>njY34GaL
z#wnLYcXe{JZTFwHJfWd`Z*Mik&q#VpGOYxy@}{sXU{>7&+IL2~KzyrnEx!=2;P=O|
ze+VW(9LZlF1+n$%h6O~UlsHb)T>d-ZMS
zn-ZLN`4Ed2`C9TnS%OJXYvyl&{wPtn~3q8Y3FF`&9^;rf7&)1bUdxiuj{+`Qm)Jk3d*m}GA>phLl$OCt1EYO
zhRZQThOVxnk8txlUCzJNoT6pSqgqw0TXu3@y}`$5{JhZcB1pw_Rw%XN@hcT3_h7mN!Vk
zLH_u`mn+5lYU8G_CqGtm*;lJ78yD-1lY90t9r>BA!?hsDxb)>09m4_}0`&L|ME{3X~y(>x0GY0yS}RyBtp*u|RRTDr!Id)t7U@|E^jcJ%I%$Tmnq*>5Mhr|7LS2X|+wmDO|Ca2br)Z*_BP
ze(Ldft69TAtKOQU{=3Oy364~@zjzSgK6hOP9)f~m&&W4l*6Et?=I1K1=YCD^9ps;o
zEfqQSK(@ywW{s-i=C|yux17E!-MeG55}QR*xOqPA0wQFY2z9gUFA>Vb?G2HrXa;T#$C_d{Fyul<~jdDMwD
z)LLCbN>K!8N73?!HA2@(U*G9kLE5bnLvYLvOB<*K8xky~nYsU1BS%i>n@9MvczlN{
zK9Ka--4I%=4kw-1wfoT6S}h^6EfPOFz4O0AXXWDRfpB928_8^}av#WJL|)W>G6ffd
zV8jsZBBYGp&z;{8&|`9anFN9pNS)*qFX3Ke@q-mjl4GIbotFnsj&YP4H=8!s3~hJ)
z%5$1g!eOOU3hZob{paU4dhwko3?8e0reU5Toos7wgQ8uj2aK|1&E|jCsSlIvjSu^7
zUw8>`C4EE5H*3o1JgByFbhZWI!zBu)lSR*3n$M$Hud|Z1>^hBdCOD~Y_1}*+y|yy`
z?WJWNXLau!8Wdf{NEOwNpN*D2_YCJtnYT+4<$eyGUwUi~c7(MUKb1sRrco=vRpQCh
z{4Id$+lR*-odu4bOh72s5vZ+w`mfLAJpA!du-UFVP-`Iw2*UD61tvgK3;?+Wthj#2
z4os6#CW@`)Xo@=vM3X8E^%RUzS$B2h{Mk)St9?y%kMlyyN&8bLye>YJZii#dp9vN^
z7v{!xq-Q-t?>OCerQcJj-eFsF
z?Ddv5KR64Z+d~W$8Zs-6O=&6wWhWbwU>BbT{tLNZcwZwBQp3*fP}Y66d`G}Ab)NVc
z<9E#h)|(Q-wPZwhsLV`xlr(Q?4HfDh$(s13=}~fLnEm!QYm?kuP$Hq%N?ZX
z=(jUgJof$j!qY@C*#z#hIywisT(5MunvQG=Kg4MrzX)Jn?ZV}uoeo0P4&TXjoSTKT
zf8-iB1!O**>Wb0P&>KlS8ycy??e&G0x;#Zk^cBNm)#ic%_Aus8T#4m!UNa=!beBC@
zh>}W}V}fyx^~QM{7B!oW%7gC&*)FW~?t8?~-cgtAaEB=*EuNF?J&OH&{%}zs3y=TD5SY9X9tVyQKHE3<09)8=Y
zYFacJ9F$c}uk4sSmze0&TAGC4+8sPLbA3u@&0x`d?}#7k)z)+eI)}W^!G@VrCN2Vp
zCr?dw>B_#d9%=U+axqy-?@p{Kd$9kK8@)^h{gEa8GM%Mq(kh{(=@H+h`(^Y^W3@$0
zXU)6Y@)Uyaj7;_0^@k+>?C)vI<9_(nHH+iz%w4|L7J9u4C62qB7TSaIf`qn{^(oBT
zmd|ngX%q4k>NDu!51AhMu!W=THGVrmZ#v!i!&BwbLtmNZv09M^^7EQn^ZYk?Gfck)
z0l7*|kRL1I_zfW$?ViG6gq8oH2wUEb#Y6r?)3KFhjW1r
z4CJ`yzeUC8z68mut|u+*`gFNaiW;1m2;(XhI?d2)qx#Wq(>hJR!rXzhq+8GOi+pLM
ze@$^DFP#RYXF5P7`Ww>bWUPe$Lv5gW
z1TdYvfE8+iLHzOS2mlhZ%2)XfohxmKEQaG2Ox>sW9-xs}5qe9hDK+$Da+LVhoH2Uc
zV=mU_I#viX*xF!F{EkfvNn6z|kJXuyx;TY8ZO?3JVLmLw(;T^6OArMgQQQ|8R>z
zrP*DEWWl?P{tnBjc4aW8kxy1MI%h>kBdg#e=_LkD3p_&tH7#RFskIUlp}0?@>{D
zdD5h&RLvq|^CHy|q095B-C7~4nOb!0D9EN_)JbKc&Ur!W{>|A?ZXskH0QMj!0jQa=
zWQPlYp@c*R2VS)<#RM3*8JWeo=A{Z8=F_8iaBomH&iGDdUwSqE7L5wB~iBa+VW-bq^-^YicZnKCeuG}cWf0q@`=ci@UQk=>C@hlZxK^F)A6{XtGI)T
z_Qy6VtJYyXp2JgXc>+Y;gAy#ZjXTnXh90I;`}ebEmyk|0aN5&>$#z~!@lp6D%eJ1E
zR)^*eagO|4Vr0%g>a<+G@J20+=xr^Y=cnmHdB9F*c7N#E;qP{|y}v(eUcWn3yIpHx
zo12?k@Od#&b)Wq2aNNYFBtc^4LIpXfSLB{*-xCU0*VraGJ4hSV9N_vo_Tk!hfoHSF
zjhjiBMD`K}W%*mIi}oGx9>3=-bGgtWn0*sLas3%@@iu=wTQRCzEVrB(6kVmM#EG>g
zz9UTN!^kbih3*
zA((}LcSfqEun_`#;G3sE@jqd(3v`AO2*ASX?airz$e%(aysJA(+H{&+)dOqU5?D@mUTv
z1p++w2qNHtfM3eRYmRhP5^T(rCVN(DUgHCxG!Y;{@E=@&XY$RcMueIN$lN}NP>}#I
z`xZl>2xL16^cuXpy}yAqly@_waWXm$IQDoln&2t^z+0G8zps7xk^wJCHf_=Dk@*@+>Gw8h|R0!kTrBfhdQ2W#2cJRRV6&
zlI@^x?sZIM1Z)hmFV9}P)_N=Nm}=6El$oxsF7JQ=(Lo`ScVh
z9*&Wl6gP&`OEOcq^}EwlQ}0i5eYI0nez_CPfCcH=zGHx|M%{uGVmIAj+>%4TtK*F;
z9f-J*9juIuj6E-sf(a5~95tsqtP2wCv6QBpg!x(^HP#rbv{;s^!9A2eVC-@uD|NEY
zXx~ZeW5JG-`!I}>ML-jTHpsWu0$|G4&dtRYp_S!S;0N|zaNsgCQbI~93>d4bIE{dw
zo}Oi9Zp)4otIc{=ZQStsOdIO7=oOfgn`@(~HIF0NnL0o)VXd*5$dh;A1?QMe7490IWT6T1qu+8h^l|
zh~B<^ibO@jijv@g5n3p~^&W*{Z3lpi9rwtEnG1CDpB>!3|HLCGdYD3&X$&Ufd0=1(
zqp!Mo%hFYfR45Fk+4h`(Aci^8i_Bma#F;wyJY}ghPuu$19`RibsLWlOs#dh0_MoXa&