From 685d5af0a3d6ba0e60e89b6ff25283466a394e6d Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 12 Jan 2024 09:50:01 -0800 Subject: [PATCH 1/9] bugfix. passing test_lora.py (#411) --- src/levanter/lora.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/levanter/lora.py b/src/levanter/lora.py index 97eecbe22..0af676ef5 100644 --- a/src/levanter/lora.py +++ b/src/levanter/lora.py @@ -500,12 +500,10 @@ def to_hf_config(config: LoraConfig, base_model_name_or_path: Optional[str] = No return { "base_model_name_or_path": base_model_name_or_path, "bias": "none", # TODO: support bias - "enable_lora": None, "fan_in_fan_out": False, # TODO: support fan_in_fan_out "inference_mode": True, # TODO: support inference_mode "lora_alpha": config.alpha, "lora_dropout": 0.00, # TODO: support dropout - "merge_weights": False, "modules_to_save": None, # TODO: support modules_to_save? "peft_type": "LORA", "r": config.r, From 4806ff40d01445aad8408430b12c6fa9a4c2e32a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 12 Jan 2024 13:05:41 -0800 Subject: [PATCH 2/9] Add Feature: Group Query Attention (#410) * finish gqa draft. untested. * fixes. local tests passed * correct docs * test for mha/gqa/mqa. remove broadcasting. --- config/llama2_nano.yaml | 1 + src/levanter/models/llama.py | 32 +++++++++++++++++++++++++------- tests/test_llama.py | 26 +++++++++++++++++--------- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/config/llama2_nano.yaml b/config/llama2_nano.yaml index d7196c59b..c3ae4cdb8 100644 --- a/config/llama2_nano.yaml +++ b/config/llama2_nano.yaml @@ -9,6 +9,7 @@ model: type: llama hidden_dim: 32 num_heads: 4 + num_kv_heads: 4 num_layers: 2 trainer: wandb: diff --git a/src/levanter/models/llama.py b/src/levanter/models/llama.py index e96f23c8a..17f6d04cb 100644 --- a/src/levanter/models/llama.py +++ b/src/levanter/models/llama.py @@ -47,6 +47,9 @@ class LlamaConfig(HFCompatConfig): intermediate_dim (int, optional): dimension of the intermediate state. Defaults to 11008. num_layers (int, optional): number of hidden layers in the Transformer encoder. Defaults to 32. num_heads (int, optional): number of attention heads for each attention layer. Defaults to 32. + num_kv_heads (int, optional): number of attention heads for keys and values in each attention layer. + Setting to 1 means MQA. Setting to num_heads means MHA. Otherwise GQA. + Note that num_heads must be divisible by this number. Defaults to 32. activation_function (str, optional): activation function for the hidden layer. Defaults to "silu". rope_scaling (Dict, optional): dict containing the scaling configuration for the Rotary Positional Embedding. """ @@ -56,6 +59,7 @@ class LlamaConfig(HFCompatConfig): intermediate_dim: int = 11008 num_layers: int = 32 num_heads: int = 32 + num_kv_heads: int = 32 activation_function: str = "silu" initializer_range: float = 0.02 layer_norm_epsilon: float = 1e-5 @@ -76,10 +80,16 @@ class LlamaConfig(HFCompatConfig): KeyPos = property(lambda self: self.Pos.alias("key_position")) Embed = property(lambda self: Axis(name="embed", size=self.hidden_dim)) Heads = property(lambda self: Axis(name="heads", size=self.num_heads)) + KVHeads = property(lambda self: Axis(name="kv_heads", size=self.num_kv_heads)) Layers = property(lambda self: Axis(name="layers", size=self.num_layers)) Mlp = property(lambda self: Axis(name="mlp", size=self.intermediate_dim)) HeadSize = property(lambda self: Axis(name="head_size", size=self.hidden_dim // self.num_heads)) + def __post_init__(self): + assert ( + self.num_heads % self.num_kv_heads == 0 + ), f"num_heads={self.num_heads} not divisible by num_kv_heads={self.num_kv_heads}." + @cached_classproperty def default_hf_checkpoint_converter(cls) -> HFCheckpointConverter["LlamaConfig"]: # type: ignore return HFCheckpointConverter( @@ -98,6 +108,7 @@ def from_hf_config(cls, hf_config: HfConfig): intermediate_dim=hf_config.intermediate_size, num_layers=hf_config.num_hidden_layers, num_heads=hf_config.num_attention_heads, + num_kv_heads=hf_config.num_key_value_heads, activation_function=hf_config.hidden_act, initializer_range=hf_config.initializer_range, layer_norm_epsilon=hf_config.rms_norm_eps, @@ -123,6 +134,7 @@ def to_hf_config(self, vocab_size: int, config_overrides: Optional[Dict] = None) intermediate_size=self.intermediate_dim, num_hidden_layers=self.num_layers, num_attention_heads=self.num_heads, + num_key_value_heads=self.num_kv_heads, hidden_act=self.activation_function, initializer_range=self.initializer_range, rms_norm_eps=self.layer_norm_epsilon, @@ -264,10 +276,14 @@ class LlamaAttention(StateDictSerializationMixin, eqx.Module): def init(config: LlamaConfig, *, key) -> "LlamaAttention": use_bias = config.use_bias Embed = config.Embed + QHeadsPerGroup = hax.Axis("q_heads_per_group", config.num_heads // config.num_kv_heads) + k_q, k_k, k_v, k_o = jrandom.split(key, 4) - q_proj = hnn.Linear.init(In=Embed, Out=(config.Heads, config.HeadSize), key=k_q, use_bias=use_bias) - k_proj = hnn.Linear.init(In=Embed, Out=(config.Heads, config.HeadSize), key=k_k, use_bias=use_bias) - v_proj = hnn.Linear.init(In=Embed, Out=(config.Heads, config.HeadSize), key=k_v, use_bias=use_bias) + q_proj = hnn.Linear.init( + In=Embed, Out=(config.KVHeads, QHeadsPerGroup, config.HeadSize), key=k_q, use_bias=use_bias + ) + k_proj = hnn.Linear.init(In=Embed, Out=(config.KVHeads, config.HeadSize), key=k_k, use_bias=use_bias) + v_proj = hnn.Linear.init(In=Embed, Out=(config.KVHeads, config.HeadSize), key=k_v, use_bias=use_bias) o_proj = hnn.Linear.init(In=(config.Heads, config.HeadSize), Out=Embed, key=k_o, use_bias=use_bias) rotary_emb = LlamaRotaryEmbedding(config.HeadSize, config.Pos) return LlamaAttention(config, q_proj, k_proj, v_proj, o_proj, rotary_emb) @@ -277,9 +293,9 @@ def __call__(self, x: NamedArray, mask: Optional[NamedArray], *, key=None) -> Na key_q, key_k, key_v, key_o = maybe_rng_split(key, 4) # reorder heads and position for better training throughput - q = self.q_proj(x, key=key_q).rearrange((..., "heads", "position", "head_size")) - k = self.k_proj(x, key=key_k).rearrange((..., "heads", "position", "head_size")) - v = self.v_proj(x, key=key_v).rearrange((..., "heads", "position", "head_size")) + q = self.q_proj(x, key=key_q).rearrange((..., "kv_heads", "q_heads_per_group", "position", "head_size")) + k = self.k_proj(x, key=key_k).rearrange((..., "kv_heads", "position", "head_size")) + v = self.v_proj(x, key=key_v).rearrange((..., "kv_heads", "position", "head_size")) cos, sin = self.rotary_emb(seq_len=x.axis_size("position")) @@ -305,6 +321,8 @@ def __call__(self, x: NamedArray, mask: Optional[NamedArray], *, key=None) -> Na flash_block_size=c.flash_attention_block_size, ) + attn_output = attn_output.flatten_axes(("kv_heads", "q_heads_per_group"), "heads") + if self.config.upcast_attn: attn_output = attn_output.astype(x.dtype) @@ -574,7 +592,7 @@ def _rotate_half(x: NamedArray) -> NamedArray: def _apply_rotary_pos_emb( - q: NamedArray, # [batch, position, heads, head_size] + q: NamedArray, # [batch, position, kv_heads, q_heads_per_group, head_size] k: NamedArray, # [batch, position, kv_heads, head_size] cos: NamedArray, # [position, head_size] sin: NamedArray, # [position, head_size] diff --git a/tests/test_llama.py b/tests/test_llama.py index 15a5ab452..7224a3ac1 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -131,11 +131,12 @@ def named_array_to_tensor(named_array): @skip_if_no_torch @pytest.mark.parametrize("use_flash", [True, False]) -def test_llama_attention(use_flash): +@pytest.mark.parametrize("num_kv_heads", [1, 2, 4]) +def test_llama_attention(use_flash, num_kv_heads): import torch from transformers.models.llama.modeling_llama import LlamaAttention as HFLlamaAttention - config = _get_llama_config(use_flash=use_flash) + config = _get_llama_config(use_flash=use_flash, num_kv_heads=num_kv_heads) attention = LlamaAttention.init(config=config, key=random.PRNGKey(0)) @@ -181,11 +182,12 @@ def test_llama_rms_norm(): @skip_if_no_torch -def test_llama_decoder_layer(): +@pytest.mark.parametrize("num_kv_heads", [1, 2, 4]) +def test_llama_decoder_layer(num_kv_heads): import torch from transformers.models.llama.modeling_llama import LlamaDecoderLayer as HFLlamaDecoderLayer - llama_config = _get_llama_config() + llama_config = _get_llama_config(num_kv_heads=num_kv_heads) key = random.PRNGKey(0) llama_decoder_layer = LlamaDecoderLayer.init(config=llama_config, key=key) @@ -208,8 +210,9 @@ def test_llama_decoder_layer(): ).all(), f"{hf_out[0]} != {out}" -def test_llama_lm_head_model(): - llama_config = _get_llama_config() +@pytest.mark.parametrize("num_kv_heads", [1, 2, 4]) +def test_llama_lm_head_model(num_kv_heads): + llama_config = _get_llama_config(num_kv_heads=num_kv_heads) Batch = hax.Axis("batch", 2) Vocab = hax.Axis("vocab", 1000) Pos = llama_config.Pos @@ -222,7 +225,8 @@ def test_llama_lm_head_model(): @skip_if_no_torch -def test_llama_roundtrip(): +@pytest.mark.parametrize("num_kv_heads", [1, 2, 4]) +def test_llama_roundtrip(num_kv_heads): import torch from transformers import AutoModelForCausalLM, LlamaForCausalLM @@ -232,6 +236,7 @@ def test_llama_roundtrip(): seq_len=128, hidden_dim=16, num_heads=4, + num_kv_heads=num_kv_heads, gradient_checkpointing=False, ) Vocab = hax.Axis("vocab", 1000) @@ -279,7 +284,7 @@ def compute(input): assert np.isclose(torch_out2, np.array(jax_out), rtol=1e-2, atol=1e-2).all(), f"{torch_out2} != {jax_out}" -def _get_llama_config(use_flash=False) -> LlamaConfig: +def _get_llama_config(use_flash=False, num_kv_heads=4) -> LlamaConfig: rope_scaling = { "type": "linear", "factor": 2.0, @@ -288,6 +293,7 @@ def _get_llama_config(use_flash=False) -> LlamaConfig: seq_len=128, hidden_dim=16, num_heads=4, + num_kv_heads=num_kv_heads, rope_scaling=rope_scaling, gradient_checkpointing=False, # disable for tests so debugging is easier use_flash_attention=use_flash, @@ -312,11 +318,13 @@ def test_llama_configs(config_file): check_load_config(config_class, config_file) -def test_pass_different_length_seq(): +@pytest.mark.parametrize("num_kv_heads", [1, 2]) +def test_pass_different_length_seq(num_kv_heads): config = LlamaConfig( seq_len=32, hidden_dim=16, intermediate_dim=32, num_heads=2, + num_kv_heads=num_kv_heads, ) check_model_works_with_seqlen(LlamaLMHeadModel, config, 16) From d4be2130f1c4671b673417dc511666a72aff0001 Mon Sep 17 00:00:00 2001 From: Ivan Zhou Date: Sun, 14 Jan 2024 09:15:41 -0800 Subject: [PATCH 3/9] Add a post: Fine-Tuning for Semantic Parsing with Levanter (#404) --- docs/figures/finetune_func_cm_full_weight.png | Bin 0 -> 60728 bytes docs/figures/finetune_func_cm_lora.png | Bin 0 -> 61136 bytes .../tutorials/Fine-Tuning-Semantic-Parsing.md | 427 ++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 428 insertions(+) create mode 100644 docs/figures/finetune_func_cm_full_weight.png create mode 100644 docs/figures/finetune_func_cm_lora.png create mode 100644 docs/tutorials/Fine-Tuning-Semantic-Parsing.md diff --git a/docs/figures/finetune_func_cm_full_weight.png b/docs/figures/finetune_func_cm_full_weight.png new file mode 100644 index 0000000000000000000000000000000000000000..9e04504e621b9c82ebbbab6441802af14956a05c GIT binary patch literal 60728 zcmd?R1yq)8w=VihHz*}72GXG*4FV#HD2+&mC@2Wh-62SbC?VZ~q;z*9At*?9NO!}T zkKgyNf336sv&LBatTVq?kJ&9=uIdT z+Byyv{G@twpc(#8&{|sETG{-WwVl4D5$d77wS}p` zU1f0?dws2k5bd&_yZ4&}1GbBipI=@opUWI9D>fj-vR%r2`(-d&(>9NFshr7H51Rsq zO?&>NVcMU<>=oL zw_iuWKbC%Gv-R>$jMt?%xJ3lXr^e=R7OTwM*Oy7|mGbVugSo!d-djF&G_YPly-3ZFj#O}*q5-P0RX^dk1_KT5Rw;dc^DVfX3o##uUr zkg2?Ckysfj;(0YvT95y(+JQa5e<9jxZ&jP%A~%YNh^VKp&&%H*x3aP_&wj%oB_+kV z*KBY37wO|1t-3&}n|b;TSSqAmP+5SHbDcEZ)<7HM5(kT~XXWIlHc3Sz)`<&U12ty*iMq`+#J0e)y{qKE1eW zmcn4wSwiga%j5kG-D^4F9GXe$`358y5{V~MO>f_|JxK z@aMIsuD`4y@=9^Bw1%)b1_a2e)sd!nt&V=ey${(BYAv z*HFT~eMcqo^mt3&`Rrt`=f@A}5PH!G80GD4>lh&$Jj0gYrlOYv)q{RbA3l)9h`CUQ zF+DsqsA96UwY_)$zDIHKF69$9*TTZWcQG-6<8Ehkbi%eOk;Nu`Obd&PjC_2m88+fh zy99N0b$KNvetv#39udOU3-6SMl`*x;ttC3bImm|sD7iQlc`->ZTtE{S7yr0wbKxbd zjD?Z1n{p2yTFK$#;!6DbYD7LXGCEo#t7WtFv+4BY==IKTRh2At#W8v+DqIc@4iU#~ z9QimAi43Jwva~8)wLo}YV-uHOAwGYG#mj3FmdVYV1!x?}U%tG}%gdu(`N;P8ZeQBN z-ZHD_3%`F;Sf(5(?>eiJIk#;c4}32+(~OLa)N2YL$HB!76{2NDgJs^!KMl7=o@{w} z**W5GB`&h=^D$e!sT- zs{q^8)m6z)nVD0k^3CVZ%$@P#;sc8u8insn8(WU*cl$Gx1p}sFiA)XV8w%L`CKIw- zWskT0-Or+~u3r1&hiZRuib$?=KzR7i>*H5Gefp%!L%owAJMUm*Vr-lsBREfAxZAL6Q>3abaPkzQhciXM#6xlER32B`068wz1JS zHxKbAy?&X43;sQ3VDDPE1XC!nNAd=WQNtz_M1z)e*f&Ona_# zED7{o+~{Vx||n{_;3CoS~cVMStzDg6CH3 zZd-E?t(OPSH@&6wtF9K)($eBK`JRH-)DP?N1KVRFMn*=#&5iu9>+(}aYc(=aH;fWs z<$h0nU^vqnD(L(4@aU+awG{_8tmGH9Y`v#Xz29GZU~sTGi*jdsoRgrKe3zS-*E=?r z?w0vjjG^Yn#>Pb+1Mg$IxtSSncz`Ry!ouS&Llt&h*4EYvHLfRUvlCJd+I}z1&57OI z+3FwOkM#8P;}NBvk(pvHi2t8;s)S&; zCKeXnsD4p%*&OPZucCx-#BIB*j=BvScy?{0yNq6-FWQY>TjjL3ezG^-Ffu|5ANs6aVVk3SN8Di(qq@2p8HA4?$?pc!Moq8f47zkZ zgcZE8KG|UY>W#1q)iMiHl6mu5%^4A_S9?cC3ggpim!r!fBD8cu*0l@0$*<=+sl0uC zr<#Lk1`D2t)K|h*U+76hKippQiW7C3iqLiSsHqXpRQ-a*&(FVqa8L(p_KHmK2V#Fd z7&Y=USdTrKs%$EmD!4GENQ{LSt7bq2(+VPsvbL`7oP>k~MAJH$+sWBkAACyA8Rk}D z+vPW3jXEy#@{&<znGPk#qG4ajMlU`Kkw&(j%~2FI=V1g!S8NYg+1F5A@d_s z)vD8NZA^4|w1N_1v1ofZ$6HD+9X<`R}mk0JKxO4&`1th}C7jQY`<>lkcfANErR>&HM=9Z~@ zcD7mE5fcvDP@##}bW2FGe7xBD`g(1h7d8*fxroal8N7)~wESMac#+w^zW*(ne;R&!ky{UiWYEqibVvy!6?XE~ z)g2LB$vRbzJwJcmh37_sPlesu)ZVxoSI@&Vcr+e|&8OQ7y;sJ^rXUGj5)`EJ!Xota z@xeSj+0$j$E+>QZP!~)mOg8teye)a0iKHU=yjk)MA^$F?eeh}6t;X<3mEg=lk8`CZ8+uL`Xohv1KU(dFaAH&~m z($96beS=8QTj^j0N#&A&z{Mv|oe{Sb_36k6*^KkG=By_L35kgwJciAe7#Yu5&2>tYyc)HDHGCk_5=U}PdD3dFP!%!q``uPW;&R6 z4({2J-{rY#rGspciP4CAs+lV1i_TJfF)k;W-V6mV&WwnPgoMziSS0AmvAK2)WY@5%s4QxvFAT|_4lhVvazwX zxI$XTgDt&TT=2YmZm?vJn%DReibO3(D{83_i;$YRkWR#20d+|(UQD~xZY>T%^ws^t z!?}tOfVv;K_7aKcgm7*|$-M~<4lg-J0#Si%2zfMAJp=;rve-R`_7xidfFgSjNG$2) z*9MIA^%)zRn%ailPIexjb6BXxP6rRy2OmCq)RiCC)1v?&n8maPh7Ap+4_j=rnBRO1=kV~*Q;vzW_|+(@d5yzX z8tOI<0l|lohtanlLI@>$HC~;amknzd9}q-6Y+kW4^HkZ77lumABT{cebwkHqq9AQgL&neG;32{4Hb2*N4b<={#hi779vbhTz z4IkwVSrW%{ef4B4&fq>j(BVNO9Q#cTMN3HmnrUw zh#NHGoO4I^jKlD7iTR&D*xlXTtT_jp>jDY~eiqRk@!`W&0YO2QN}0xGt5+>`PC`OL z^$8MQo1^BJiQ*yU)I}_(hI(h%`q&4Le5UGLZN(TN zB_$;i{}v;~cTY8DblpyhZa0XXyTrtV6~U?P37Mz!+qc=3kF2Vakk@994iX_VupVnc zSVz{nl{yJzLutS~apG>l5X&bIw&op(GWve}@bPV~5a)*|^Cl?hBQar(#iXQ1Nd`a+ z*f`|IV?fOGYu(RnMoTU(E|MGLOhQ^r$jI=A-Oo}8po7};k{-liyp@Nacs-JnlaJ?R zKeE;}Hm20JAbI8W{<^-;p>4VnVl++5u%t3zgge0ePJdJ4mFZ#C$K zQ9Cm`+ni&?UR9H#qGG)BOeEy&(ABVA?Gw?;D=tWO&uSBen&5TsqxO`tw&|;{U%y(+w4R5}${Ltu zVQGnE3rp=et8sjX-oCz$(6h>%GuPZp^Gn^Luke&}b*ji#gVr~k_F&&`>UY5XHr2!4 zzwar1Z5Si)8tvDwU&zBiUYuN7`m#=MXJg|N5)!gd+88sxw=(SaiSH%r$iz3mawkW7 zEne8f7AJ=`2hO){-2$B9c%+o) z`1lx!a}J78!>*{c0H2Vu^75&PImYy;$@3>(l#+{69YNo71~O$kHme z#*OAT_x<=$wYmeIkEFP`Sg=~l!L2W;UW_v)HnunAei+)f=y>@P9gj1YTFyY>>vl^& z31MMN-Mfe5?BZe-I}b@YQpkqY^=PGp_Tb*VdkXtc@@DAMJT;5tc}7=QTBE<-j!FcUcJP@LCC_wVqt6B#WUK}bZvm^zPNO!!|7y$-)BI`!MnuW zv8$`A_AqrZF)@~vx#FiZYCSHe{gfKD6??bLze;1)B}Kle#${q+$}1{rf?Xrc@wi7k z++lQbTbUtkYq;5uq0rq<^yT$OF|`nxH|FErrfNOVk@y0Vt1MFdh zOFW3+ObX|EQ{QT&^Yq4(oTY>P(gW)Jnd0oWu+29MQIVPKXt(<(9 z(1)u54IN$Fp&DJxrK_8x_;||Y%r#2b?p0if+l?DH+P9vmwqLM{<1$5dQDaMsTagn= zp|=?sVXAD6qWa%IzPvu%HUXG6AGS60>ldT2Fpb^nh@!o98-QpKRk&)xto51IvLfs0 zM8=Ow2hQ8?UGxW5kPH|>Jx}l4v#uC_h&z+}8`EXG;gum+CE1YpX`WK#_}ukt*AgG+>U>s6M1$DJ^Sq0; z5Ri#$%V6c;uV0g`p;utPXG@xIz=odfs|kmoIxDozTQZ$?iI$cYb$gpq#^C@21a6l@ zYXB?BkjU`*agK&v}=dFv!?aQc~M1!@y=LTX{}a zM=R7T?Rnho8tPFqr40+7nD{8vxXYe)V$&?}Pj7Dwt!n2dV-%#MVbeqduV23w40kyr zbO5FUK@~X4ZF~FDdcQo_zf0Y z9~z@vD|3k?W4m+-g%ECgdjSe|brgsm+snUhj>mlf`Q;s%sm zabcWG0Zbhq44_EJ$yFy>LY=ehH>N)8)`+Q?zNi1`MwP>w(_1bGspGCk94&<* z18g{+$0v^?g@6OX?WZ9i3#K)-v|PD)b1ST4AVV1&)`)6m#JApFo|d9?cn{A0ID-l8 z#ORM7eH^yt>VExtqTiYeGj@TL6p|vIv9U2TS!es)+{wFHAj_;G_3QvhT5NU<$mX1m zt;l)#b#!!$zo+;g?^J(2>8}vFX8A|Ubzw?ep(Cs?qfg^qbZi??4UxB*ex6 zFXvg!KRR%3Yim=lax79hI%WY)5pMHr4Zei<964RF!dGC@0H8L2 zVvDGh@veusfJ>B1dsQH*Jc<$USgSrF0^n?auz5`_OC6CQShi1{O#xjR_ar|!(BQh5!0a`{a-W!kfPO-T;MLr76VMN!TR75GNM<(z$tf>T?vuG1py&kO`qr259ALQ0h%b zN);Ek{Cs_}>g(%!lBDrLc0He%GAoeHB6N{vlKdmDlcH$3Fi zXU_~4zo%MQSt;)usC5?OTgEYnyNVzY4n!Hf!90C#etsF)=d0s2ApoJ<*53B!>WZ^# z785`atpyaVZ)oTR;TtIO?M0Wa4oK^YX>wFz!Txx2FUMCuy;*or;c`^m1AkOEHl_nO z^L$`X5GFtmeYmeAU=-)-z(4{<0ReSp&El+t>}(Fu6g48b^*!rFfR=1fBLOXV#rfDi zsFJ?zDvqbcLJwoz=%_YwDNqZ&VYkiMU+n`xTAXan5(y68@LjMXIV|*-7WrVQ%Dz{12leJSi~uy z__87MFy9s1u@x!@=}1A{Txb3O5D6Ys1wh^MK|TW502e|c8QeLOdmK38HI*-F5C7bs zmew2M(%imN7^PA77HAoEYh&a>)(e-!OF;5b-k%N;w`4zCpOi|3-0kV%;h|?@!a~3x zrUj*=+rfMHB=!VLrZ2IvJ;=+?KhGo|7qcct3J=(dA>k=}i3D`tj5n|e^jhCBg7On8 zDbF@Lv$XUUkj|{*C~IiuAn42GwkrW3)zvpQ-~Daz8E1kIsT`*KEVo(m1^TB4@kq%p zNlC1-j%rV~)f0kIY#5aK7> zq~E`Px7e6!(u((aa7RUDJ}aja0YGc>gw!JV*R|ks zMNKaQ{NEs7_ssH{ksJK^Az;EkU;Py3h0^1+C#+JbM2W3sw#IS&jAZw3J#$p@Q<%AW z_4z$I$d==;>3aZSf&3^(Mu_i=3#H@vUC1Yg z=8`G#SO#C_(Rl58)bRR^RkProsA%e`akFW~uXlLV2WZ}UH-5-16MjEftWS9POeo+w z@tTy(V@h&X$ArCv^USKv^r-dAb1cU-=21&uNu}nJbdsOx@@V|9Jso-CovZCdGmtw4 zJ5@c;UXYsa)nyxX)@#4(1xr?aM?;?Z+p>i^}Oi0VQgYD6%y}Te_LKgtUHEhWkcJ= z&YGqK>)Rox=fl^^rm^Gv13TFS%K>b5aW2{Gs`Ve=&i4%El2YUhxkuqYW`xRJ*j=JK zFJy0^YQMKrN4tt6}we1LZJLA~{W0Q0a%Q-kDfuG-<#&FF(DdU{Q?6$w;iEXX^PP+G($riZ`LhuHG@aW4J#&JI+^2oXUYL{Mo^ zj`X2=25El+7K!f7Ye_|Pp$2P>4^<{j|M*oGWyYy0sY~Jbm4*xG z4~eLK4+*W-BuerMRqUltZD>_9e|Vd3v}Q+#`wj$uK?Rr%Bu^T|phjLuprNwMX=flrL0kAJ+Et z;1DO7-y<`tgHTuo04EuI1*P^Im!WVI2rzB(XSUt=a${p-?YD1)AOPQ$lOt#jr1DBh zxeDrO0{9aIoOao5mj^u{;p>5Pg|`Gqi5X&nPlF#ZS41^}H)Ehf>xGXSO9 z$DjldK{jg!CE+|BT@bK**B`XboTOJoQMJ+Dc2zN|S{=7#H1S(b561bQmM~P-CPxbN zXR4@;sg2c9lO7h`9%oq!jpcA2O;DAy{9eJO8u(DoGS+o|&Ss>-3STUH$mLt!J;J7@ z>gL5(QsZCE^1l^SWbZl(z0#`j9kLObHKV4c{;WMr8j;K?dok*T;aH<#*N28m|CShX zt^G~G&sq)mtEw;NBIlMhaGyV-y*o9G@e~T6){TrQ>FEtcN_@DN$^E2x+KNCN1FY=} zwQ@iVkVX9geWIhXva=IYQz05&QPa`$%-F@+b^=Gawv7MbAvU-6u0#~6~pK8KC za0KXdo`M1?B?6Xh8*T}tLT6xL0Oi_Xe|=KG?NlU9E(#4mBmfLdm&5H)^BQKbgG36a zp`TXUQx2?w`zVV;?p=6X==9J;ME+T#cT+`@JPhDu;%Q{5F&DkK+t!b1wQC?ssuw*a zVpZ98uky0W*`hr*B$gc(PLPxi$)%yqh(&ih#X^_z5^?(Y<45#9KWG5$uc?Bdnx<3Z z%Au%|8R9@1BB)G+>~l5l^eaS@b)%XKgD>yp^!YuC@z3g%BltZ?RyAy|zw%unF!BcW zHB(i$LUPEbmLJ7RL*2nGQ&hyKQ|%lzf6V`C3XQP*s9glfaxmJ35YU<^wR#)rAUu#QPcVC#CtvU5Hrjvu6 zA(hAnaDY;Nu8!VxCMHf~8JUn!*_rl8k?Xbh?=SGU&fT~e^=a~Wu_lk*`Z80ucI@<_ zp=v`?@;f=spK|aL$px?dtVHYEv?)Q54 zwcf9B!k@OEc&Bqs&#rVEo_Xa2-;xX3CU<ybB4Q#9gb$ipCxXNi;J8ngG_4wU1~+Np6C z7=Dr%7$n%GRZWaqxWPogC+w4?Hvi=F4+jb?A5$z8?_vSYm*UucN8!uoB}noN>g0}H zrY=6?IHmeooVppaH^Jbf_$7zGGa$;X3FkiD%8?nrR{LXj<+4qHmX?J!))N|agwIa} zIm|n{x|*YIrYmMxlH*=cHv4xu{J6}azqT{JSnqH%X!G%t=a21gnFcR*@Lv|?AV*ya zV+oIdNFhDtionNIaIO#nSY7~+2$V|KfB#@R1}MESR#o9FupN>1+#P?fQy zjP(mey73ULpvDxH4|(=`Z@zdps-&bDP1H21WDPq}7swnjax&tYhbiS>ns7QMwIM#! zaX_I;&+cq!kmDEFb-PzPtd(hqq}v}HN4*jF%AhPoDKTD4@;ZNQQ{VV0hPJlofpd49 zNd1q`TR2e_TA2!}I+m6f=tUi`o*gXqT?hRM3yf4iSK0KRW?q3xwtW0CmkdvL+*H>} zH1L0E+?O<{+e!dVgX!xhCJj^YlxJ<$e&H)KVL_;;TP0`HpZ2u?16*whow@8TFs6UQ zJ@S40+;>czWPEOEi@IE-icTu5x^m;@O~v|R-k+R=(h=1S`cF**w$jh)f4n_2Q0YPQ zPvV3upTY|@4O$U<5=aa(VB~-TpZo60uwfnA5Ak*U4)@@lb11)tyUc}q(dFkL5u>9n z@|#nBExj1giq`&J+@$33n2eeuUDt<8{dfJ2^K#j z79NRcXlPs{{YoNa*>hQi!;8Z%B}+)QCsl!vI3D74J`` zCCn@TnaUbQj)bG17M;Hizj2GTv$EE^H& z6Z|HTpFT|iHC`C6sp(J5_>djVpD5jT_p@T^<#L--`p2BQ;X+qG7TGNcS(|FmEva=z zl+Ybm@q6Cglj$xd?8!H=xq7YoBe&#_X2XYq|J=-fL&bR8jbp(;da;YypKje=VI^N5 z*tmk%hbd9}oBY9scHg0~LIs{L(L#_6F~dF5tb6PMiv=T9PLw%0ITm(yevs6`w*!t& zB6M^wk%+d|*8PCweNXO-w1#dtq%$n3`t6^T#Y`4p{7WkLFof zs2R&Yc;F*LC||hr{XtS|02ckrcKv?>vcEQ3hpw#bjhK!yy-shIvPa{2uLz6Hf4y<{ z_rTGtAqOUKktanQG=XZ%l&O5?(`In18Fs}8LKW9DCx^5B1B(|ph*2m++PoV?EupK+ zKtVwPLn#hY`D_Pbsv8)f1o6NtD2R}qot^t>D~W^R(#lE@AmVB8p};hOBN@w`wjtf< z29@jK;`Kt4J}h5fUxaQzE|{2{^w4!Zz%*+A;1L>n9s$o;S?q|<=;zO$d0)T2Z_)T* zhNFvLLqs<@oFhsb8^bNgeG!ZXgB4B@$aMC9c*;%~>v`_V^}3!`b^RebSMAB+UkqJ2 zs~4mn^RBD^1-kwsS8{*m@5!mi($AJ!#)iq!&-a5(pfEo?x3loW9QB*ulwjVH6DQ^t zey(m+qyMzbwKrYYpL9NC{Azo)pz}k%Lx=fe}tA$QPDJf zs{$Rn`l_}bX38}i)b?zp%Mk^_IKb%9Q}A3K=31QY0px10O?YoG1-=!ep{Fqze=rNzbsL zh6+qp!MG$ZuNK?@R|8~s?$C?B0fcth`oJ+mish?))?eG;-!FsZt)^;MLKHWDiHb3Q zN#1SK<(Djy7ks2Q_?Mw}cJILhsp;k*t5`LtmpDieprF>%3NQ$S;2G)s=x7=+&&_Uu z>udrF4oQX?;Qt7|1+|)sG-u|Krv4YMf$al9bv}p#Yp!ezm>cZ?3(lu<@G_FqUAaHK z(=o4JVj#xUtwTgdcRGeBTMJ85>x<^#&jWzRSy|y-Npq`=|ZoO|Q7X zRFsyMhL>AbASKnnIL+ZzBKS?}fhwneOKsXAS{xm6yqb-vVp#`91#*Mx*oBFDi%- z0*<`%O)C(w-UI|lg6dJpapQ)Pm}%K(5Z}=*T)04{V~Ealk1jl>IG9L_t#q_KkCkii%=jVCapMz6RyiBO-zl zCc+1lRK$@8W$-&-b30E)<_-0Y$rc8;JK+ADl}dk$rac-Xm8yL68XgnzsMBu!2mf|8xhQvD|oTU1c7ottlb=l9&LH)gFna|hd_D6!2eD}WV zD2q8cQsOjXYUBSjSQZxg5%g~9?;bSWh$w#0_3bW#VF{z;ev`amCXRY?%$!ZY@i1syuLiopZ1{h{|!Au|8<1*-*oF^AGkFlJBo?vj1@z*8>&b^s!6Zh4SdIb ziw7{z?&>H$%6@&~9DEP%sH-M__e}Xf7^a+BT=f47xU~;s5nOZ!=jQ|@)W}L z@P;HqQhmcHW=k-AsL<6^Kw4>b8Nw zWzctULA@Fr8mb>0RG(rN=MWX8LxGc)5LzVslUas{mdE`ZtR#8#)=5l9^RYQOXL*co zGk|eXmhO|Rm4$V^hvD^^T)xTZ6r%g7DOanTerJeFN-5)uUhk~W!(8gAJi4?%MSTv1 zAyJ#=piHuN8Edpzu4Ura7yTDc&|ap8NI1R-XzcwGj6dI%q#Ti>UgJRHs&RsU$5x zq@*MieuKdKz-DUK_~tyQ(>m3!TfmDCmMf%+4X{EX-$2^**H>w=QINHf+B>-Nk=`Om zs32=6fL{RR4n81|=Ih`CwG}$EP+A8c*LkRedV|lfUW^fI;1&yiUF;j@r0TVFkj5-o>TMypjeeqr8w}M#o5M>k669E#~ z>*M{`4w_n8v3|3!5XqQ0XEp($K=EeOJ%7#+*tvNV4PJs+A zMQyh-L~FK%v~)WO-<3HOOV@CkN>3dQi>WwDI-xlai6a4h;8BfZq0K?*-51ijcn3w*3->#KGfo3js=$7>j1RUthgyfbiivu`*OqmE;rRflh$MftZ-=QDGvWnI)LH1+`2(J>7D1GrFLN3bz; z30?#bhU4xsF4T7sM+ub58=!H3>}TH}Nc{sJ2Af$qEwG-Kj*d?K$&+&^a6juqJb{0L z;Q$-Z(#P&S@Wo%@<0FTPcqjiV(i8=qVBq?4_wqt_J>HPvH6Nn|KOFY@Dtc^iFcEAq zBqF_h`SMRmW`q_6jRKhPPak5}uCA`go6m{BZ7?zQeorA9c2*LYUMy^En&ib#Fu-yO z{T58)k>_p$j|3wqV4U|Yskyl!V6~lu<<|c%r-L!RyIfbd>v4v5_7GJ-?FUzz>Zjt(~2<{<uzWekk07Q(77cZhvnP5b7tT;VAjTCXX zae6S%(4V7C1Cc}um?X&ik!xRp-yo&EV)OA}@RnSLbbuK0?LR1j1rzPt$o#}X2&^?wiocR zC)2@@R;-kv+Li8VpqvfPFEmp+?VhX$;wX2sC(kSGof)1Jp9N`jWX9`;cA~f9qu$)Z6m_HbawvFi(DYk(|H+$s z7!*w?jgnV{5+ukjPXZ}Hz+v+$1*aCy#3Isd zp7UT}EL%Nm8DMr}1_A{;2G1-k4Z~owKf%-csEPb<64lTFh10*Je(8{We_UwF*qGV4 z0@VPjyG`tRs^1?aX4~Hv))KZ-I<>lwO-T?->f1ka=c&=N`kqDtw(EqF_p!&UXAKLU zc~8vfo>wQWK7TDMJC#T#b))icwp9s5bw*B3icz7HHLI6uY80F?;(jZd{FB>d9XI}b8kl8_WEbE(`jFC~bm$xVmPV&VM9uhZkONx5)HTb_ zFVFQbq)2a820M7B+nFaj?|8;O@AtTb6Ulc41JF@jrtZ(grx{90ekq&(z*S5--!zKy z%?8W|D_ug4RqMXdaa6wy`Zm0mcc}|KgwH^J;|z8UV#GuEANYA)-<8P63X#Jmk%T@_ z@C0HXy!#DBaRcZ=jIYuLjD<8S!#|*7jAh*t!q#A!)%DZUBXcq~H3EDJ_F70Ae;iyQ zP+SKdO91{t6ci;RqoW(Tx`?3T-tBk;Tcg;NrKb6DAh-$}fGPt+RtGOD1g$iwxRbz1Z7bNe)&D-NQ7uZe82>{o_HFMm`z!C+c;R zjaUeT1l{4|hCBUSWJepXwDorm7~$ z;j0A5;p-jxCA?>maj<5Mm|g2DEl75VsRLEp3Mv~f$v-oefUoae!Bv0oz!5{)Y>bVO z0bnn}e3NC)Xqxu67fDAd9ne9$087+I?i(BGWM?(7|LdUp6x1C_c+J@rF)=a3T19f< zf+zIaf&xN=bZCxOyWksDKpAO$Z?6R;L9p(HpmFg-<07>95X0X9BSu^d&?oX1DrW~G zhRuNzK>HE#2wiJesXE~OzvSXxn10hUi%lK#k!0LhxEmYHYMrznzo<2p2WcuUpP7@%+-6DN zTo7`kCNe*uI$!ZKS^YIl;j$nG{m$v0x_8M&{$Pnd=KHDf+Ru@gnf>NiYrCV@0yTD) z2X|K$ZBHCo;@rH_yYJBGx{|cxcmFK}FT9-0XBsQ|^0+s5@9fmIE1nb&WjtC;m-a34 znyR{NC@zC8N>*1pUd-iBO>TGCT*+_qp+ZsIRMMp@Y{F+z%zo(naZq4k+5ell2A#S6 z6h5C3*U9esLCrUusr#it{tM;cYTMJTymWO*TwL5z%UL|=q3#6l81i)wt+F@Vd#Kp# zEz%+dG6pm~WYiZz0Ed;tJTL(BdM)6uk1h;Dft2+(*leM_x6S4g=!BgPW=3%W&Ke>_ z@oGeV@^4!v``Ra==l)02U*Bw~7qLKl&)Yn}EwXfIf9Pv3e+V@E{B`qx?Nj?-#4z?h zKI652h$-8*_%uN%h+&Vz`jGS`l?60}u9f+}r4^oVA&krI=x5%nGE8AYo=Nj2S^-F< zCs^(9psOM7e}l@GIvT`X^+My&e*HI>Z_Pmy1#o1f3nl+=ub~*_TLat}Q6~yeeItYj z6dRkD|4lP+t@|9{8v%{n?cT4AwQ6ovzHm%XKDt%vx{}$ipHT?qg)FUVjB1YPB-d=# z^_R0=d|%wa>4|uNy5fc3>PulInKYEITiDg*C(9sAdS&!05nZ%OOF8-0tl8qC36ACs znyO*GyL)Oo++4=&Q{#=Y7gK)^{VU;t^}-)p(d+$^+YTZ>Hz8e96RZu8uS7c}Jy`O^ ziQ-0+rx#U-yw17HH}@#o`;{3@vUcCle!kuFm;h$#76y1rGAy##@ZS| z-%)bAn8`{mXD1QvVyD{RnDJ@qcx-sID$Z?bjrv3C9;Sk=!xz}OLFazkVEGX-0hJgY z%|LeDE(c92(Zf`b2a;pn;xozNeGnr1Na?agO>|a9V@uS9w=&5_F>$;|dXdkD!jm$q zIeKs)l7Uw(iCJ~P-Kdm+`hc z-yI5H4B+CcWw!+Su!Vk%^1q*q#mIL}TCDq+Dr{qW16__#+nan+c=A^Q&oj*6hS6dE zYk`$|>SS)`fI>?{X3#S>#-X91Vb>}pfn#mpFa``%Q(`O9o(X+<=6etZ`f7nK|>D(Ea@w-7<-|BIhlxFDY(8Y(pL0nms3sNLOW00oFw7HWg_ z-@ji_PLoqy_;_T$UkJ_=I6DAfCpOgFfrm^Z2|~WxY=0YU{}Xd_Sje@8hYe4U&vv>h z$D5I~1TE3fQk)Ep3DBKkqYBJJ?E=eh8lX?aKq5O;Aj!QstUXN zAl&@%YBf%eqs^|X9~GKxJx(y`D9&SB!oa>x8Tjd<=VLAdi}{~62O|7AcN2?rDN9v8 zP5g#z{dPJa{r=V?{>q1uILlH#YgGW$0r zQ(qL34Xurqd4PpWCF`(e_r1}wzKK!L@PMHAYE|I2^+>5JVbPL(JHq@GDvGaj%4mP! zzVV|OaYFPC)EhpmOi8gTdYj*N#`)3OCO43@g=lKU4Hqdm>BVN*cHE`hK>sI`P*CWYzN?`@ zjWpYV%`gF!5EKf;0_3nFI0Z!#dK1K`x`9X`q6$*217r)s+5+WFq$v^Xg_2U!qkETA zA4D|6EW?R3$jLL%u1KM{4@lY)L5p+i?}!iNu^J~z^z=$SM=)qa^Fm3*0V zb6;Yl#)>qN?dNaJI@M9bN}1x4@1?bhREXZkMiFbJEhrS{(g+$R0xzFx8a3PPcVe*70Oh z@Fz!nNSjqSh;4jb=RxI{{uY3lXm2l0y%nCTY&=dyJSFES#Z@PNKGS9&04YRb;e!Ep^r%RGoTc)Ms_ovoA{Uut7*ku6T#n^jeI%offo@tH*gBuH5EzaFrMLz4@lI1R!7D1vpPd#KqPm-OAc zcM0p%qPEBLzM96YPAW|e*u-Q*^kUAaXc6DOKH-cEmd(VIH82r4??!N@jIpX^csPig zH^rz%&v3O^k}H(FwmLZDD=M^@t*_ZMiW7BpAK;2l*7A{!S1)kv{g;x!L5QBxW5hH= z<-!5@=Q<9W8aqMLB54arxiMds8M*M1lCttS)Qu-y-R6?#_m7+7rX-)4ng+s=7zWFO z`OsA62d5E1r!q7FZ?v_L+N=zbL-=WgLvIjW+Gi(->LwOMwmYzC5my4xKKV1yU>UXURbCOCRW5BkgmSBXvp5!o2> zksch>vjBSWqj<3(=nT|{v(4g*bhNccX+`JEl~w1126p`77Dfx$TyZsh(!PC~9{RyH zSYUY$7 zdM{q!(a^Y_Pd3+kbHgZ>Y{+_NThG*h$=tlZlH+Ht7S0t?We^OfB8;pQb0n{Q_@bZ~ z@J>~2FRqH)$N&Y+jFUR~7jC`6k&%_fa~|FhJ^YVgtQh>CcW8fwWZeAQ2$_2hd)3npL6j{u!iUsZzJb7lwr&yf$%j52VJE-iVc8H z(YoelEJj8~JpNyY6lG6!$f`0I^VNqT~3{!Py~Clp6j~6 z(`PBkmODZV!tA$y=czo}qTtwI;v`}V=ct1WHlZ{|&Zg#5=h;9Y?vFSbNkiQ5KQtJw z^qr1Wprc5*v*1LYR@^$UR&qdT=e)>ss-HO;o)a4lq3_=l48J$xelf+{|2g?ko*3wN z@RwAs%PcIo5cV3654L*K+MvWq1ARky-He>=DjDgepIfI)ZnKnY_*gl32$h{2 zXCrKn^H%dg_nkZMC*l=moefaeW+(q64V>bdfXM(mD-%Fqf7lq%34*#ET*3AC@sox2 z$=O^_L9P0%Rdxf>p`bG%M#N$B?MT%t#BBhF0POHtOw>w_)Bi=QOT2+-mx8Hf3OYnP zY(Bwq08O#l^3L~Z5SoCb3(|^l(EG`=A?IzVPy#)LZ<^qDvljb>)Zpb(>Zp(SL(4yN zTkP}xH^Xu0q`;`4J-ITA_OX&VD z?%o70$F=PjzDp@3G)EpYL+KOJ0+Ijng(8Q}x>zF2kY z_#TvD6}~<*{m$xDtlWyiUn`P5zf8sOvB3Fdv`ApDwylq8p#3$ulDkI?H>M2+N0gs! zFa71I7i4&7NI~R}Ig04_WO}NvRb;JRQ2Wrr#635^y}4F^V8xFeKKO_E*y-e)+|+Q- zgh?UKwJi1Wx|>UO3Db&%_#*v;y7-(%?Z0YZIZjf{E%s9@js0Xk;q(67JJHQ0g_=yb zQigE87PVuV3@I15sh=OBkI!ML=gi!8WXHlI+AD)0&`MR=C(QZgn%Zq~lWd>S&C?I3 zI8jbm>dY2t?A#rNX0LMm&L8>roV@8D`|e17q9UAddTMk0{bRrH@@!Bi7kmO^BP#CVQ)FdzpwE{~XZJ`E=AG^~nL6tX`|{rDCNstW7~6>-F4AJ4X~ zUWqs#Y6`+t&*PXRzIwuB!3|CbSv0^4Nc63weT&S&{F6*@pF>1d4$hn$%3}baEqsYv z-`97R`kUvmZtFDzjJZKYfAo9&!Mix4Be#F5kiwe@*S;`MA|BVVx$gKecb$ z+1^%Pnf2+K=G_4nMmgg$ix*#@P<948bPRr2rhi029Ua%23-#Fni8nvYM&{l?Ka%d#THo5Wp-Ml68Z~94hi+-XWBQHsBNQq zx%tJWp|n8eLm@2k7YfQ=0n(6#dl5Ba0Xhl*u~9?4!h{mYks?b_v6L_9Tk!)y**f8YoU z7SbviL^(v(;7Wj^1Rx&=w~0c*VYt5LJ|fYcMhK`o{`k5dDZjiFM}hwg zmQ8H-RN#myC5Qs9L88s1Jg6&pfHotEVAaprghbNFA_)?IjwEA3RY*c0k*h&XIcQ?i zRmHu0IY}}rnn~){p5{Ju_%JblCc-5{J$|b2c1yXZn`B?5DU9hs2}O#}XscC@deX(j zu%AgNTDf_qfX3wK;?t`rSIeUKQTh*OXWm+3e3&SmThn8Q?&z`ae$#c6kahpG({Ouh zyVmaj= zRq#Hj5D1Nc+w&B`9^phT3s<>{iNc#xT#>5m3(gb`ljP={Vc##k(2q4b2Ho@XWauFFa@ej3MRv&#A z^|-R7(Rf*&JddKCocX)2%JM#rOH+3L_TIpe0&`P@%=zGL?;;U3r5k?yxhGEr!HS z@?M)|I_l!)rj8KZ#D(fFYa1G#c9aK^tXX83P(hv(7#hlmaEPT$Oc!{{?|`6XP&oVa zsVbcJOn?vc+>vO?4RDWkD5GZr+60mif~Hm{Sv#wadfF&E))bcJdut@2p z$dSE7TP$mToEdR`SR27fdJ5QxvV`}q2%{sX?|dwE`Lw3ekqgBZuQSSx z@_U5NYR8<-ZOYUsONo4{dpBkOhV70cne@}sRUVN-odRM-3yah%{U9!}2OX>3e) z_+@jV|JAK$FK}IO?s{oe_3Zoay8zA~u-_&W!w}?ZpYjCn>I6ajcBZh%b-+sy+R@%_ zez3ru87T^gL|qECe@2Y}iJk!?`O^2Klhd{%M+C)9r`tBI!=BEd@LF74+hRmgzPU(8L7zizkT8tY}FC_ehqO7idTe<1Zo)}3g z5HSpy*;K5mhx)7P{^7c<%QmK-HlZfE7|`z89{AdxVeV@9bUU?-yQ4%Q*oV8DU`hs?vWYB z-XKx;Nu%?c?1_cwVbwspK_?Ojkcha3p$G|t@f|c^AZhxM!$BgUK(BRGA#i&oGV->8 z*Bh?iqlE2?8?+w*38vq-qr)geOaywnr}45UhJJiA)=H3HPWe1aKtLQ}YS>knK_UVm z>A>tpCKLI&oo%l zl1&;16h9VPOK8P%QMPPx%iWib5Ro#J@ziZ6Nh8OyIZ#aPN}Ord!}~ilomqDCZuPU( z?U6e57N+s+1;vR5vG*mVHk5y}-8WP}N$NK`ZGar9-a zBhD;Y=QRj5Rn<}uq|4o+CPqF>; z+s-=p9ul;spT52-J=Le&WKF+3{w8VnY<+5(-ci8JG19^mm_z&{vZdm$nPgcW<~ru- z0d(Dl)?@c@5Q*(05lgIsTFOXEg0960BDHk0;+4<~1eNig>-#`_93%}}J&b4DoXubz z&~*4C5p(=-+P+RtmyoZ@jS2b(4epGb2r;}OqKJT}b45k`L(fgF=(8p0r-v|%#9P*? zNm0aHVK)unzWi}UC0mx7_SQ4f|-@|4B+ApKu5XWKQ}*o^oWXr zXk=ozL5v_VlH+AbCYE&9{>Z1b4}|tE47Kda65O5GLoeg4!Pp7+X*$G zBho4lbKK%zB_ObXb=~f#sE^2-vuDp9DEfAqE?ZDSaB2X%1ib>ig7>);-R;`*ar3OK z{4I|Vs`i#ACvu%WE%bA1TD}h(oX!FPJr2*13adTY&dvJkS9&7!gWh~kwsf<8db~uP z{xohJDaIi2gD`APZCQ?J8>s!&gcTPC%zg5?5c=9`;eii1NZ^)s1gr`rA5=F>h-Vtb z1`6GuYc1EHa>cJHe$}Pd-!?Uogsf|yhIbMDAHEwhuMXa!VhIMp-03v&(0QJGhLqi} z{PvYnu+GBIhmO`NQ~(l(f^A3G57d0k??+1C&NWx61}^8sLy$ic%I7=HjK5#bLAkyJ z#?4B^fH7>6ng7{I5~<2)rrAXozN1rVMMgUEXBUaxa~tpTnDs>f7B1WSiA^vAe&(MB z+MP#HTUC{U_WBGiMBm2ky6Hok+MZ|I=YQ@^=SZL3QW0sEd$QfX3h!MqcO%Ke!*3Q= zZSL}l=wb5}g3J%`qzrcO&s~3@`TJTjcZ^+lOmf=6he0G*iX91SM&BK^82QD{o)p?i zRu9(GMBnY}1eHWa58zlL>;)n+%Ob@x|3|%c-Coti7CImt78-g6dkPlE(qdm4 zeULa#olh5K-xZj@=u78&lbX@5AU~6i5{V-N3yzLL_$RCl(T4W<)o`w84K_jqjv`Es zmO{jmI9--NWJX9Rq`QiZQRX@Lb48KWKD&qUn4CZ&K^KyU6C{wGGosF2JKG)CDm#C@ z02?ToxS{xK_qGh-nXgGP8jW-BNw%{QTWBQ-uc2^MyflB867SDeae#JzEhrI&f?Dip z?Cz%X*Um4@e&=>BG|IHb5G8{l@II-ddi{(-MQ8SHBiN-m!iiav26Bq!yH5j3hpyFE4`0<={IIy>#P4*Rb4Bd`uY+1!07v$V-C)h(;WxkZRLw$A3RN=3h9Cx`?A-7B-+OnN2^%1P z0TfB zCBFn@QUny7IQxl#8$U4!W=4{}fwyva*Hex83m;QMF4MBe{jd`8qVgh}B@RvGP8&35 zGYhZXq~A{u{mQX_f8ULt+}HlL+@@;entC6ezu{aI5*|9{zTC_+A}d%eySJkrw=+O1 zYCI&IcdOvZE5C2mC45a!cXy%bcpSH=iaXFF!WD0AQK9zxnks_DLy*WFW9x}AC{S`L zZC&-}J-tBOH+Hs&o7M%618ft+?ghYi;Ot_OMd`kPbZ%j*Gb{!p^u&XiN892ibH@03qzp&8zbm)69tyXwz z16MQwgU8qv2(E5zH^~(FTYtuHnZ&#s={kq&Fppr^%8(^(o3@u9V0Y@zOlGM6WNBu# z_xT!^i45krrp(h$<#e4dl7m>(?!NzH@nf}qU46Iao<5y>V|vJr?+OG-kSiT!o;r?exap)$UB?HZH5e*Hm4 zTanxaKgTUv<8>Bu$(t^EHFU@*>6G?c4wrNeyC__-lwTe6aQn>eVXq$uwKr!FXa2xoQ~U##j7?5 zu%4M3SJQN^-5MS&5`rSDHhupFqI^koRJ@LSDY#2Rm<4u};8cYGF4yC`2&CaukR}q- zC90=NY{*?zYmkeIvzPRbsIa=Luan*QJeSmV#c(ab%S%`LK#{JM$oU|I0_2gG?&?hG8hFarYxu*4XSk(2Gdg)fP8K*Zn<1xgBe#M6>~_WO6k zW_xr&_;#epGdgg<5*E0LAs@TRt~BhF6h9HeH28@Cj2kPOL7)LpfG8V~cx#2TYO;G< zU^iLqkC2=J^g++T8iJC;pIL@_2v2Zxt2@PLS!v- z(x&*Io|JKs$YMLMU+ul07e|kDG0O|ERa+%=AU<0!NH7ksJ9@BOw>^*YE6U5uF$Uxe z+NxJ^n;5|)Z3je0-ZdT>M+yR2m7(tL8pW=_LF6fgi)gkIw1yK`3P7nFj<0vdiq7o%tz2X-9zJx!_>(5mXd18UiN>vN9J# zj=q2>0uV>FY~@PRcGtUc|A`u6-lzN1Yx%5Fh_<@=zSb0&enFNV3}zXh{AB#ceZRj$ zL(7Y?{bSc-wF|x6cG}k!dC6BGPz)wDkfd|1EmBg#58ETVzJ5I~d!n1-&gpDP>Lp$(%#=MV%Ca`BJ@j0!4)6ctn~H835dBzkGu|0zmn$YEWXZFCW$>)Xvv~QrI61O zu?Qwa&HV&&tA3kI@?d zRI~}e1>jK?P~`j=s8t@`xV?7}?Nl_{m|0jk_S-?L%eLJaTKEyX`KtEsE28%X@bOF#-c{4E(%RD||r@SqS(k=$eYg)RSHxOa>w zWdi<^c8sWyFdePB0>2VP=&4SU>LQpbJ27ucc4U0!*}*5V|KMB~Ds%|cHdd$0U~h5k zzFt4_7_<(+r4?61_C@?rYR&IWq6c^EcqS=&By3&y%FZ7bY=_d?G~n1<_~-3;$xK5< zEFhqyjJ{XNMY5fmOch~v)poKNA~{i;5Ksl1;^Y`mEHcmo(_qA4i`2pPO%7=+_Gf6S z377+zjVQoK7Cxj$j<~@D+bPMn^&dn=FKW+3yg7IgXFe}8U$<@@I0~YGfKhG}2#Rdy zF3rGGU%!1rLf|>zZ%M#B!8ZAwK23#xNm?yq+%*h%&;Ly9zYz@?HT2{2&3XK zYd?tBkOYR17-3**nCwG`vmJe-6^RSpv`GypX+%`iS>SdZP(eV7VUUsC+uzM2axG5y zm|IcqfP$yR%cm4DAmxRYYmc`q3j=sG@p$wyc(pkJ-TY%g!ilqVg5s$)v$Eaq^i|u6 z7Fn_CM`k_V_O$j4y1}3;k2VdcwUyove9&Q7%zUij$Mn+!jZ6PLw9dfYOG{_Gea$;A zbzySLQ`AqCf}i(+W$fwfeadXe7A?9@ysb<|TzudGwa9VA29pU0m<)*O@hsqW1W_hJ z?lw|}^!q0x7HHR>X zv=(`X;*?H8kBxY#ZP+&L5g$4<VEx*?^j zK2a>PFXKFzEpue>l1K)OxCw_|>Qn?%LH>g8Byc)ae;q!HUs7@zTn+HNkdGsi$4E4V z%C*sY3~qv7;sylz*fyVH*jH^an7bbAl#GNB1d>H07%1up;RP}=5X*p|ec%*{0}C97 zL2javZo|Jo>m1|vSpF4D(>3_^f-RS9Zq9TMduTT`#jD8n%&@-osvqBN(~UNAI&@lE zSnTI{ec<7aam59<81@|={+;0>*cxVMOriYp_^xBKG(nNo&MWk7^djj*KlTVjT7&qE z929#T+Hc~o)cscn)32kk#_$UOhNMghaVZnR)t?aHO;uTyB!4LM>+>YH#ScgMzO7HY zor`67N-;lbzTvrZM|kMMf43|?DTX~D&zHTZlZs>e*NKai^ z_4=jQ3f8MenXM^15&diMR!^esRI~p7V!_7$GO0xBO_-nE-Q6{KcO<_1J1>BqaI|=Q zd2Ym++Q(0}^zU^m4?dTC>djWSBa-AdKVE++whU0Co8On>mE1VW&n{N2KkM4jG2m~d z9sVX{K>3$5B7sw`ZNxIWN9%x>` z89lz)#n*vPN!b|JV_GCw26*^39YrRXtC9 zHZfdQXWG#i=bAOqqiw|Vc)8tsx3gL)Q)_Sct7}}ZfE)_bSt`yk3m_avPSFd}yE$YupAwo9|M~M8tpv%1m(QeNMwDv5qruDbv`UHZ z-0Um{+iB>mInO7s`A5&`^_eE+?{ZeFeOO{rlj_OWyj3Rg;WdT+Ek919^a_aJTpVh- z2@M#MhIk>2w1l|c23ZPe2C4q!*u~+1@fx&HD+^ush2C-kN|HG>3G^P=n@S+eM$HbC zpP217Z=Q>th42(`1961|N+wDGgx`}{5*Rr637rpd@8cc;RSbY|q#WV55DcJ0+=lX% zVXK&#+P?~u1yqX4?x=oc@~&}HM@N@SFkR1OR&f*Q=(&hI>614m?sAy}tkT z+I4HGDd%TrtIkYp$<96Xivf|AR-gS=_O_(0ezv6V!d>0YWAv$6`D#qWQjT0RiXWg( zqPmWe9rbO}m>#K;Ajk#AwJ`yGgfDO*_ZE9XRZ$g)rtP{Svi8{3e-J8MenFw2CniH! zfyoiqH3k>mz;q&v`Jka7b@pZS7CVVzxnb=C zVK1hQn;s2$_h6W<+Wd*P!;CR}(|jcHOWEAnJuL-Jyxfn!61z?`ZWIEmZ){(?6c2m> z#WF#WmU4Xh@Y*RRURtaUo{zJ1f{O>kuW+ACJalxcu*w@wWN0fRW8w2`ct}VYU>_(@ z*7#lla87|q+=N8Z;b~&fQk2*76DLmO=AO&X;QzN?Ey|%1{Rh&%ay`ez@v@NNz<3KE z+J9UDP#DR^=X<>wk$8naQD!^vRRw^1Rp_g#*9H`s(YI=Gd`f zq-+yjx}@K^-!*F8Sz4GMvrJySRgnH2#V^Wk>y3AAMUEmpAynm+Nr#quB4{^LHG+c& z*Y5`n@boM|3sR#VsZO|Nr3inJs=7L-4UxP1uM&EA_|PGt@Y5FNxMn) zaF#_mgp5?j(bvPT@(g5`E@V5@m_rVtuH=qxGTVC^?MB(yKYs8nkLN?RZmH_Bi$zN$ zR9?-MdFb%H_{#-2__L1J9z5Q5pK7bG(s~P*wD~uA3Y+2YY)g`#bS^YhEoZmS$YYR0 z2W5Phe8{mc0}_}BiM2&QniY~sWk$yry)vF*3^9g!{Hk%m*2ga+U3FrL&ztNo@ED%H z`|Qr*AK0S*yILxblYN;vYRNO6wZ+f^JLC9R$glft7K;wQk4Wrd_bh_bKoG=J`33!G6GI*u?gD zYf%ub6a{&-UNc&Ad9)If$fFI3-W!!Ayoj-?(|x4YDlS3MhhEh7Y9?1INWkc;JuWv*YUkyOklaP6vPhzT|3* zw1G5aqz~J8Ic^Qv>;ZP&Km+8Cd15CATQ8vaUX6VYl8hZdI4~6?6vSc&(kDpN38kd9 zbtR~#NmdQQOc&#{#W5MYGP{ z(aI#F*XWD+K|oVcQQ5prUkXkR(qutUD0r&CJIaP9$J$ZL@c~QYAMz)Ur5qc4k?=V=3_Y#US2gl5J_owO%G* z=Rh~o$u?(2PxstnY%@u&OENUpVu*`vvK(Wc2i5cE(rX7+&z_ozcN=&9%k3-@{%3iL z9QXbDZV5tiAO23k{b6EEQO@73M1)>{QSd3hW6LzIhtNy?ysJ;Bz)U;_t(UO^49O2_#0(y3_`{lEPzxUf6nAxDaX^*+?f2G=ammt0WD%WMIfx`1d z+BprAmQx8ra57(d@F=2T^~JcF`j@^d|1T3q`>IU;B7p>=u3QwqfAcNx;$@g4;dB$g z50Y67Q1|NvB}#)zn;TWN8E69;6@Xc}aW@QJy?uSPCh4d^!yP^5k8-}@qR|q&Q2m$u zm7YAGHx`w1rubXQouX?sg|6N%4kmo8K~tZ~Vl^hmda74iy=kcJ{no_w(YUE{`XaKa zDm1yWkA2EciFa7-AY#adapb#rso7a@3?OP>Mey=Y-jL7#Dqw6RyzchgWQ2cA3iOT( zJiKVSW>Edrhdo$Pq&r(tlrO3O`mw5Q=f3i zRyXNp#Iy5KQI>2AD>iU#-A}hX^6)aLA1_!ooL~D=@|pRoBPhd`@*6WYa;;u#@Gzl` z&*#l6Cx)K$dDSD3?i42E4d&P19<0AJ=>4(D)uMaf-Qlj0??Ph2rT%@NbWKggU0kk| zU*FVaqxq4>E(FzED4E>_4Pg)<7|a^D7#SIWoFSq%g~^7|X?4!nCn$LBM#8qPE_SPRo4xIkLf zjpAaK1*?)Y6O1Y)6HmeWK_#T;foaA|6n~BBlZYXxKZZs|$^frQC@9&~tm2V%ra|={ z{T6!U!m`TSZeN?Jupq&ul)ZSd2xH2mPhE?Tmvm^ACtJ5=HW%=$a$K&dv9V!}@W};v zvzdr>_qc>iasHpC9M81kXw^Bl$mHc(y!xzhbFGicA; z_v?qHG-wY+bOJho9f}O>MpTPmaj{a|X1fryGZuI-m>j(liCDsgiU@&*#%d071!UC$ zR15PU3bva)xOtNX$KA5f3K@)-15RgtqvFo`^WxUW)E)Mn5#;0`Z^A^Z^L*m zVdf-UWaGbQBB=(U_XV`wSI?3N#aKyTYOk2=xZgOVYlMhH%UQCFbiIc(YkN(Uvq3J_N+3UXDIA?$)K%FrSTKk&I14=d|} zty{MePy#N2sFh0@8CCEns3?%CLB~uOD>5(>)QG_P^;#a^z{3*hBxy&0dk0}k2nkiQ zv>bWqv3~!38ZvwlGIC9eaudB-( z5)yJKY^Mv;Emw~BtbSK2{h!KIV^`TA{7K#9}XkqX;8T0KHilK)mXC~%jed95ozH0 z=ro=eh?&Y1*L@+{rHwo9}pol!?J*Lfii9*LQ~!? z3=8reZ`(G*nwz^RKqJVuXxXo;?E)3ZM5-Gf9UZN`wTlz1ug-uR9UUDuhN4>A5LJGS zH7v1KYuB!oHoFZ)C-mJ9T6!>Om&#qynPrl#nk zS#N~BK!C|X$;jX1)++~6HS4TIE}d#D>1R6}Jy32jOGRq{eco%r;T|SQ4UeIJIx$K?;v{8o6z=-PmPe z0>(7$MabNecEEik6Jjx^ip)>#>mfuKHW2AdD{Jd@8}47dbjcrXk=RTO8TW^=_qq2q zUx(EV*l)BD5(YK2GQK+*en+X^w|8c|xBIG>8Menj3?OW2@*G~EfAHXWh|kEROpl$p zmqS8SLZT8AN@}0xr~17x;Qs4a{d0c>geBB>27hf2d=kj6_4;Y9SI6R&U0r44XWmlZ z($^*js9>Y^t+-AAa{@S`Fxpl4O@cxw-dH01f=+A4{{72=dXgM2!arezF1)F@G)&6; zngi4)9Nt|L_aWmpz_TNaM|cDEJB%4c51E8+nH;A?TaPbIEWn_8NoKmFYrg@}DU*6A z$AJ&2gf~Y+P);~P7)@LeOIONYyjnkXA2D0WYM~tTn01Q4caYp+=qQN?oBS-lo=_7^ z$D#+L9)qkz0ynN!Is3dI47J9>B|PV_Tkvc>xflQu5!1mYdoMlR9CZNnR?-u&2+qk;bZA&mIY$kHy@&5&A<;;fi-mnszp~i=0!keOVis zw`gW!tQZ72SwrNjV-}@fQW8I$QK0j5-cN{|78p&%JThAU4A*TNC6@`QtoFhT{fAbS1um)2}I4LK5Q5MQSLaFo{2~ zeUf^w#AB(Ks9K@mJ;V{}5sR7_c)OI{`;Ex&Fv_FVhcL{-!66)v-3mOlj7#q_?7eeh zAb+6m*j@sW8AC;UT(yOsZZ7=|i=I4rGJ(9e;n!Q*3??=q(B1-Qx z#KXV{f4$5*8d4H=ZAfwpXIuLDXJ_c%IXX1cw|V6+*?jDW(N5R@P{zEwOR`->IaJ#% z1{TD`#Ff@oFx7LW2jmQn_W8KYXjVqKibl+SX8!usHc`E8^~8j)H|_3wadX>K>Q#-A zk_)q*XU>_CR z{AV=Gqvk&d#aqiiG~IzvZK=8cg!zB#AB<4tkD%xE@3r$d!0$r+{c=p~Iv1KpV`0g+ zYM9AA>197ZX6nnN{%26AZT8#%Q~)RUDpG!DhCSrCYB`6b@YfwU{(tZv)Y}gZA^`Jt zYoz8Lx5gQKnKVn^gsq`9wF1KtZ9(q0v$u9GUAau|lICPz{v)pDK84sE;c`#SY7EI( zJ@7?sBE3nStF=&Uub1rd@Ri;4F-DKt29!XuU%M^Grlg@tM5`xL<9AIJVcTg zxmxM3_d`*bD4?^+W9s2Drk0OVuJ5%HlkcRz4-+?<`Doww;bDdC&7Q~U`_D33hgrR9 zn2gDDWMA2>s5};}Pos4=@5gjDx0zU6IP>)j&))L+`saJ9oL|B0AJ5aO>drpbEzH-T zFD>TU2zCAbXDPWW9@>52AYwj9c$R>U!FY#NIDdYVbkLnZmf%ZBx+z)G06er|#-4&Y z{v4DEL<>_lig1Fn=m#-CgmlHg`301e%shOMHbc%82$0Mj9G0R%Cxb+AgXxzpB_nu9 zz7F~b4W)Guf@7T##s~U3SX$1zwRw|Xgp6wR^7e)r_AHXj0)j&-&GvpFi>Cq+CpJ z<~@At){y({Cu8pqxLz~2+Q*vObSWU@?j_BTP)r(OFKFvr&?g0O423maZlwFRl0#|# z@pyk(do!bg&KMX4nx$J?E?d0D321evX+^74b%Gh zt{8;aG8uJh-Fi?VkP=ZSWH>0P>6WaNKNlrtOih6>EX(QBj=t}BC>Zw4hE(+>6vvY% zpP{Q9{+83GIa=)g5gZ51`~m_U2v9_WM%FT}RNdTsm7sPU4aKJB2G#C(nbYVu3NDOX zW;o$A{xoY5^x?NxuI%bIZ;En_;Z~$RvCpnFIY<_rc2H`pQTK^g_tIrEC)O?(@U4U$YWRwwg;jXT(WD`KaS5@ExV(5?} z9i^b4;80lM1i|`1%&uOuh6ZKLWmA0sDU6Jvnr2OSOcVm>kh?&dK}W}|SKpyDaJPdW z&maJo5o^^YcUE%aMz5}(i2fm;@$31ajwjEA=C2c6(tmbC<&v@zzgZ>Xhth4d)IX>_ z^RcA6r)QVCs~NJN{H|_kdvlE)8HKUoYW%u^m8h#kf5}H^7<6QFULUnwIf|NWxYcv; zML}3b#(=Hacvt4VI}kexVB$`9GbfVHJ)Wk|j^(Z4(cd6->}^Jnr`yc-Pbq_m4Vgm&0AA*jsNdW_cwNH+ItOuKB2Wu9yb zK$=>Hsl2Gdb=R#6WZ$`UyuUL{F1?qV>qbP-y4}~Otf}%xMx61HEZPNE8Dg{^W&hPl z{y`Q2M})C4nuDb>%0DSaP)@D5jf>z`)quhnwPCjXM>$ABEKvKB%wB*kHIOosv9x#{ zA&1+BFkdOi!%5B&e9B~^>a#H3P{arC;&aQ`4HO5qS4>5srCzW=7ps6MCvdiaQ7z*x zB*Bv;!7#IEihCA7`b+r3T z+jQ16)7MF?-nZTS)r6(AyaIP}_C8jx9AOT~N7Zy3-4c^iH~!80MGjaa3KRg?6R64j zFuY+m?%bh=N(94_BHvIYU>+MH9#9{iCQfs_2-v!5C?qHw;0W;zw0e)5;J3s#ZFK)o z@T#_I0%YSZ3AG&1-M`tf|L> zsVJ*{$sDWoq9{k(m&R;xJFSy#@T{C}Jo{M3;?GJxG}(b`SGapvJ*)dJnUh6L8CyOh z>!Qk^9-jF7oWwlQJ)$58aqRGSU+cvjwZ!X)?mT~4noI;tw z$G=9s?<_ml zumsnvKG&LJeQWzV=-~0o(ER53l&y^3Z>sVCkoy0B=xf#+H=z;pLNMO;%K0znfoA9C zLv*StYgkrGNbEs3=I|xzb?-!kQmb8mU{Rq`Y1gjkfUuh!f3t4p->Qv?_QykZih(U) zH`j`>SZLd& z%QCKQ4G2rA``2Ig7QSrX6%+yisKQZ#alj=VgmgjhXsbAg{&$q`2kP5ZU_=OmiNzMX^&7YYb-2?CE4PXz?U%$R$dKuX!_;hm=qnMv= z`N+H4L4T){6qznGZtS&6Aq8%#@wQn?G>ZiN-yZ<;7^98e$7VMDTL`DhI@Tz%-n3v& z-iJ&i5=DfAj1a8IX{YGmemA7=XT>&LS8Z2!a}m-}7hxg{VgG>o%eW7%qO`--L7b*! z018R;0dhp(XV7ig-ak+F+2ZjFpkn)NSVbq-Yw}L9*v*X8)9}4?`>3}eB{4M_Z z52uzw;)09)fV5S1?p#EM2`E6;5srHI6UqSh@sDnzzgnWN8UYk0w~lfV2>SH2*OOVl z1V!%r3&0Jc?dM1Ecslw+64q#TEVdLTocQ=%gzV4IkXc{DAGvLH6|Z_kc-?<}@q+Hw%d0 z??tM;msgyKVX=?*JUO?`BOOc|O4~p@0-U)N%HhL@845EnF>|1`kP$9yBA6=>p5m*4 zXKWaOh_N|CFZ9_xE~MRbF!$TBGC{_>|K^y{sZ+9uUWbPF1{;1d((z;4&0hr%5#3Xnbz6c)>a#T9P=OCT+xa4gPvhP{O>5w zKiF)tFxqOZ%!jbgflW#Fot<$X`Md~CC82_d?H=+aJb)OC7$rkcVWuF?v`lYpoK$d0 zSpf;z4TuD?}cJvJ18?y3IrXr6fvBs%^`VX7=Pxu5%<#@q{ z^V(9nn@%B_2NoR?;E8D*)5^|2fQNpbfyERG*Jb{FLd_|2&_0 zT;wl$HBM)P_`}~G@dKd(|6`TOQNMx$`E-mi!}nDK$3STZy*Yt=ZQ9*C;?`h4Q&UrO zt_%$gg}r>8GVWlSJqRd5pJJj?jc)4-`xnSI?14L!pu9+P3dgw=djcL2W}z$nNT_8& zh6}J+MnoH9@p2>3xfJ|VL~QJfBVFu({zZQ64?TBo8UAoxW1||T;2dkXv-*TQFlS`Q zDDU4NtNcQPwW6Yea4pWB*Eu*jt$;3bNt*j9zc}~2qJowLgrofn3>(>-1yF3etgI|B zgF9p41DpaLW8pC3SRjaij8+D00nBhL9^{>uSAk!jf&R}Q-v~dC{b1XA4kEii&XSB% zCk+WAg%mzfAnhjjFtU5VGo=Q`ugm8xk++%4HQvM#grEk*kiq1cjHIklRPx@(U*PZgP;RCf1=O8EJGTQ zkBOhAtv$BWt*Sc&PFfvIG3xWrE9BFf!Mu^!7Et|*c-2T^Fb9Mn_gaQ=G<*X7PQ6BN zDG-je#D;^kXhvSOu1x@RfL`H$sJqW3v?}>3MqatnfkFTkKAU9XG2=(a*zr(@FkOb^ zaMvAq1Lhobrz1F*Y_kN>uNy~9*-%%s89(%W=!GeB zxih#O6=11pH1E9Mn{kjJ2ZV3Hfdtn4ifpF+mZ*1ZtcHn5+33Jox?ee+kFbRyBm~{kascq6f!Isu@3?Y9AHcdcSTSu zC7*(2%!cnn+Zf~g@kX;B!(PfjT_3i zpBJ`RL{?t)e)%rnO#)-Dgxuue#=_VE=2a2>p2MEo^yii{Kt!e#tyq5u$2KB74#2*2 z&+FH(X*E;w@D*#GoUkT2Oc0MbZ$hTiZRHmR`vB|Aj2F))!hwej4q+SW0SSY~aK6)c=S!=qb`4mIZQN*Z<4{@^#vj7vOnilub}(Jd_YX75%gak5fA@5K zsyJ##@(l1mr7Oq3zB_}@Ulgh7IxKj8#Qnh@SwK1ySVvM$ovN{);6B}ex`ze;!^p@; z7AhoyuAn7E4-V0dJfK?$uI&;(2?vlHe*fXarvR_OCrX@{u8H z%+jAs;MvWQ0S4WvGF~`BYvbL;NzKTsed``{`Z&7DwCyk+S^g8d*RTNMR3=*B`Sf5u z;9_cc-^nn)>^eP#6`<^O-KTbQJBLQiA)FUZg0CwT3^>S3D99gyzsT28JbV&b3=)$VsqKB=AGMaV{+E83|!QSVG_P1~6TL zVR!Chdd_)J{;S0hwxR?3G_Hn!%^KDALuq&6HIUw7LHJLSZL)SPE#9v#?HC5w8}@nN zHVhZn?vwGJ+K8NaGHijOoV?@0GI-+cF71p;`Q}xc5_5+k_IE0en;WNG$a^qHPk~tw zQz6zelfCgsTj7M^^L4DIuc45Gq6q0dWJnWnf0M8^2tBa9_N-uVxB)D3a&ppnw(ydC zYrtwf6BN|sX~TagQO{68OUChrOa2oQJAjAFDRZq~ z4r?kTG_)&B$@9lzL_x5wituoEVod?4)QqX`Buvq7i-qYroW{gk0XHb+1`YufwCZ>r zXm_XNp>_>a^qxZamdd~m>L$O`m_JVzx;#dTo(4WP-B$mj@UCXd>0I6%Aa4#l$LbEyN&+kgiTe{d*v;?txBUwW{} zYdHcJx2vyaYQk26T80kjd*G{K0`rq8CgNsA>vg0XufT5&2_cS%Bw%Vd(@5omZ9oTT zA+UBvUfo*-?rilaV4yyWh_$wcU>@9>_;gfVA)}7Hs7>8-_9F419=8mqzID@X;iZ zq|D4}Geu1@{S4Wv&1|CWZOOXq9adB9p|Bw1!4S@+&+06s0ccupxP-C2z`P9}yk(G_ z3T?Q(0V%=wKCDpdc?@sVId$qu$Tp`IonqopL5H$VQR!&w{0UeR9xg(40r%;3tj5Sv6;vHvn z@+9s{FDmccoLbH@)6*RP5q5qFPDLV<8(q(tgln7qY`VH)6iDtzggnbZE86|lhG;UfPi8Bs$3#p!uf4x>XNZX!Hf zBD_WGAW*ny6l;=^ch6yT@jD%U>_2c|jNXV9x ze&CU2dq7$TI0u%{F%Y}NrHi7~Nu-=x4m&&^^;xVc zGQNr8hw986eNz1471>>!rM#VmbB}WbGLzVIyy!TZ1m)uKd+xKp;1`)gP zz7GwCX%c*auxMyZ2>*<80nLEL%=9G5iuM>E5OYzzqW_L5hD>9}xhy0m#x4}=4C#?! zBBb-gw~f6W;57A+mti~B5kHiQh!9zZOD0}C5)BTLk$4vPyoRU%EQ%CtR#YsCt4~!# zQW?<#L%ZRRyg$kfP|pz6FDLXh&}qhaolQ9XJJDI=cxFcJYtxb|1xK_%4wE`qF*48` zvQo^xSimKHh#r(3t!8Q*k)Yq&6^%S}S_;m_vp6GoU`A(`iS?Kq-@6w*>h?X<(raKUZTXZMj^JvEobswAEuM*C?yXQL@m!eY0e*{Xj$XocMHC!pDbPKN zQ79xK5|u6Bn_%?8xP9J4=1}TS+w!u1EiwuzL(@A0aAT`lj5ec1hu{TGYCy>=Wfx7$;l1~ zJ38YnX$i#-kslC&OR_~!(5*MhqF|}B2*!jM7yBqc;CBHqdpX)#`>nxv;TR&dT)ihc zf?)0-J7?Guki1qUepEt2LM7OBoL~YzzfvLYr`2rewQIa+l7ok*+6g7gG--HAPOI{thPvqF*XGp^P6p%j$1X$St*BT9$=#_2)1&xlk-TFNy3AgL%xd=U5`bF^awJciFh!tp>Oyhy?L zdEh9tyWO3grQ^0ZgkfK`@K2SoKdjcc7U&w}2sq|+0`r+2XYva0t^&Fg5ER^z_Z+Ii zbuif`MQsg1IpSD1wez{E@=)32_;@EIk$nu;69$1X+dCvP3PE) z=g-|g*+S%VS8@@v;pd7nb&Ae`10lHmNjOgGZWH3+0+JF5GlX4wnTX)g80?;wSWyt% zlwgU20jGn!=My?;5e`yUW?=h9P4sW3hzag*6V(@c^MfT>-8?GkT~d&Q1{ZeTNIjlEXV990v%D*Rlhc?9kBA#Tlz%2mXk2KU)n;aAsjoBsE@*uEqjH zVOhbJMBL9VE@PsZOiWDoQVjZ&@5cqJ@`LX~D@FGBsQqKw#yF!2YhCd%drE4I>u>I4 zC8B4P>`?a)?rnC#q;YXFTP zk(L-exfr`W89sviNX)EYkxSB)gG%o&^vuY-NvofICR=eI2eo0x&#@y! zi$*d7A@m@Q1HT?){|mUGsJhhwY?48s=lPUK7BQ!cSZY4*APK}05*9w4pT89v_RD?! z{V!kb`#c!pi(0V*wJ6083xkO4F+l5iRh52zM1DW|fM8Tj}LB!1qB*}}69X)L%Ve%~&ea3|496dy ziC_;_Jsu*85SU=}bct4XE?m0Af*e{)^bF1)3TwXHL()3lZPJWTO2FAdiXuz=8jjM9 zfWp`Q`qa|~Fv1Cm7nJJT$^~4hbw>yb>^s`%^Pe`Y#^@KEh?RJUfm|%4V3icT zw^w;da5wMCu4uFZ;!xk9%Y0ONMT$Kc*!2Tzslq4!>CyWgr=kUvl%$A)-I1YvQXm88 z0??Im3lTSc;3x@TGXSMGqQWHse z;ditl2tOZ#SS2=BXcEb4baZr$)C?h-3=m zzi|_wayUfOx;|-c2Pvt7`1+QX+b{^OMEy{KV-PP`9ySRgqxu9gye^dqkC{w_CCmrL zE)f43;5z5fOq;u)k+5-f;68iBsca)}D5M2=9BV()eFx}5z%LYa%ec8k9k(|c;NWR! zXkfT)_sIU^0URCQpf@FO7RkB6rtl=ycn-c>ZpcY0ej%v*$lgZ$D!A=Xu=(M``NhR| zPgNZR0FxJeJ5Mo6|B7h?0=}3)`ha`(;c0)J`!{Ld1PFi*j4*vD)7bEcoFFh5dx37U#z@sdGF}g)f%6t7RANJ8efosVpk6DWJ&Ir-O9)hr zKmh?GTJOU1NFP$Z4b~B_9lKgj_3$D?o7fd$eZFWc88Z4oG-?jN1JKqO84yrL)~^$v zLB+fG?=69=BX#iJt6`MEi8zqi-$)4PJqu<2fwD74KREbsSgd)HGRNelg1 zmssab&mq2J$J$AO5T3Jp@Urh(4ITGuq}?z|@3Z3$Tq?1Sh&?noVHj-XoTRJPI|rek z6)ul|E_MD;@?Kb`3kwB;V%FWG$B#4qeC}B(d4)^DH$<*qzYKWuAx{#+F9Ya9FcH$g zNERb~r*vktKbEYGsjD_hn&Ox6NNLG87akn=GB9pSLF;V+LS|St>@_OZem-= zk;vi_TyS}~dqaOp54j`ZD^|kD*j2HZHzAc|77VvnO`E}wEAR$O+h)DSD~nAl2?Izn zQt?twuhqZ%tzG3Dt|=3csY{$chiUnR`qode&6kC1;`Of0q z>gM7?z2YoTahS#!NQH2)U-xE>8ai}^^p&t#r$OdT`z17%JS7EYDO>J~Q-@2`gElNJ_jh#e(sZN z=2Ns$5kPt=6Dmhv)7AR2<>iYPYk@z}Km23A->J_OaYbL@o+&CRw_@{hz3?BFwrkM| zcwm43IEAGK2XM0DMPJ4&c~%~4u=?DVuL9L{zUOVOF$YUTam+*Wp55>j zC&r)$;U=_7$bRd=*X15={g`SJq@GYm8qIPg z!F$`ulik6AOxdpewsr1b@YUq;q|ZuOPEYrM9srz2?7Gx6XDJ1nQmNIa4#I~op7Vkh z4t2VdP3bs6E)U8>GE7_j@urjuU8K}xB8Ul?bvpONmV3z!V>yX= zm~ev1#g=xL!|uqO_?Nu%+(Vh&bH6d$D9W;7t*R@GH?c&lwUYiwx_mNqYeo;4n{xBB z{0Zcib*S}7znA_GA)>M#cO^yix8K&@o^e7Pp9gsO20>CU`$gO${eBwt!@8+vf6xuu;sRr|3s|I#>xbE0|`$nu`}MAQg;&cX~+a(4bxXlc4M7 zS3lhny)Wnu6KahrexjzyENY;Wr|8$v<)Uu?GS^9lNe2fT76MF-EAN42vQoO0T>Xj!ZabD@%h`KDSdHTc zTQZSMf3lL+BDs-I^jp^8GNZBZNPI)?ts0D{ z=$Ac=wd&aB=6Z7$;8{w9pQhdJ{OHl6U%$_Pa<$ZN&hxwP*Cst)?_S>t{EN#6S=UNx z3ViNso3tCFNVk9JxJf`Wn;4Tisw+#@!?i-Pn#}J$@BoG{zHC$)>Ci={13WY`unVUb zfplYV+;Q>P5gR6c0$<YuCB1X(xB$J$U*@g zQ_T_*rFhD=0%bt@t$uU+{Su0ybAZn6$D}G-Q*>>8j|Br(JY*BOc{o@I&(3nHOY8&Z zeOOkYcG(sLzZg`?*yj*BJYlI_TePTydc%oL0jeR+c7qTmi^`DoQ7d{)qDelA*8sD- zCL8B6B@FChMA0z}K3uJdJ?aElKs>V8F4(-m&%&=BjET?4OW%w-K4Ip@08So`OS4Gr zB5}u1@nJ6rnaL5L3W#U7s5^BUQp86{?;15=DYFqhJUzp*+H2R!s0#8XBNt>p<~8YQ z(?vgrrdg;U?%U*C`bP2nDPg;Rp{3C24rh~GDMNO-Fz3T={;UNCG38^f(Vi5gNYhAP*fSp-#pXASR=} zz)dm1eQz)E?!*~^bCq0ddrE{BV!FW8*B)U|>GOL#miV6vp#_&NQ8a%-Km+&Nwr<_O zU|5{poVj!L%qEqblrKp%=}C+US8}sK9kMl)yHbDtJmX$Cw}^BXlDCPJ+Bxb%U!Z%G zQp~@wK)uDkyV1WdM7Byk1)anASRSBIB6UhWy1v^?VRCixt36Oa?bv>6E1GD zLp*~aQsvw*AegnChCh-EQIxe25fP=o!Nctval{QQBsAQ!p+79nq${!2gyp98dBC$_ znvZc*$AG2Ox#nmC)`AOz%t&4`-&E;*da9qO#EjM(!WvjWMV#%V*4FJj^=2@)>sADkYuB$|ZAp7}843u}_GFncDvA1+r!^NQR!S%;o(T<394D`n_nvm| z`w_wKI4Bd6Ss*`nb<#s`rG^&%Sv2=+Sce zM`oK2y03{VL?l(4(st{39>6d%L*=<+13rlhcdP25H7#EGy?vVybExzUqy9osWP!2m z^wUrE)FeKpS752R(LXBPcH~K~3g0{i9>jQ;egE}P+pH9O6qD2`4C6VEX;wl>xFJzy zGzB{_V`rX0Yk+M@G+!L=a#bUaNk>!~8m=9Jx}V>l%|J+5b2Dc66<+w+p)G@$LHAdZ zGXq=bw_w_&Ns~IotNZ8BvPc>UwAkc>G-=5V+s8-OQT(cx<|wChoOJiuM^F;r!7*@GG! z!DCo4vXPhEDA*Uk#t}gfJz2)3wJue1_%jdMC`d1XmHRfCs4$>Bl8s# zg5c!+zF{by(Y$%{q<}#Yl!1Unq+i>%ZG-l*^_{d@r?sFDmi$bPBFU|xMJg*hycFzf z(1SoS7U|+J6-gKoYhZsKRTG%ifuL8DWm*XzBb^1HC`{@IAssEXP`jR+?jbkztaJv} zIQcHuZwH5IB9uSXQcass(=`1!wYt+hcy$3^K{7HEYGTfd?y;mXZYCZpSz?wR$#*s=)OTq{HE6{1KsBuH~BzNnLIxcvKeRs&!|}BHu9*rKxiUO4h&l( z7GxsFVFcaM?Jd>=lpGr4Z8&T-aoJ^iZP=-+A|f_lp6gPj-eQDFg!uN8fBcGY4(GSi zp@>K@v3sbm4$13Em;rJQ@ovLX*95r%SkU0U9GWy5IZAdr1b`ZNDJPfbvBgUXcD&ONxTd6Vnf4krEUXgq+xxCQxSN`6A9&iZ6ftyr%VrYulkjQMt(G**a|&Q@2=v za>vKmuiI4qCeuzO<)rq{@^|lY`3zH6)kmqiLfADlzQ;K170#w!6Bjq%Ha4h{WD-Q* zUcyyPKWs#j!Ie{nr!f03W#?n_qwCnxdjRg2k%iAipq^7Zd(7XYi9UyZ%gjvY443IM zAy-i#G}$4&806E&?YYS1(~dKWSbU^nZa->v>5hH*M?u%Ft96Dftu;->HOYQaQ*g0Q*4=t}m?p`f_#3AV;h-9jmkV1- zp#kxsG;7ujW9>dNVIdo4rOyB_r>o#;C)AKnJehXvvneoI z??Ns3I2^7|7GE%KcjxKTr_20xPNnp1KBYT2V~rGo#dH7}Ti@Rh42%mMj=1KAaP6zA z*P+0dyf@L=h4b!hXxr+ysIN*1-@d&u-XeR}g95T>s>m>1&W>3;peuW-{w`}l7P1@! zSRqAkhg7=y!-o%xscOt@BD7E66A0R&NnZymNQ1GDFKk4S81I^(t;+(oXF?sbAdMeFkpEs^(KwP;vmxuR`dj&4--@QSY2KDJO zgglJ0WX!BBrEN!QxwNXeM@wWD0@*UIbh4s^5w1|Jb}P_n9K)nSh-$=P%9C0NaAavd-MwTkfd)?)yZHXouvI$;*YgIB%DV`bI9h=4E zTCF0+h4QoL+*~OHj1=OCGzRB_&by)DB^^_#LJY~598>b(B~Vl(RzUSgBCMPn>xV)> zr8V+9j*8T=pIcN>%A32(u+s>}2aPS0eQCkZi|puB<$Fug`5HK@^?fAhD{d@&&03*u zX+UAVdNL!1cXc?5zYeh-%(4~qTERI1AW#c{z7R=?@I`Lt94urQIBthru08oy87!+P z5>W#o6Am3@9v!Le6!s%V@GFDas@qp36wd+)?y5WkF=e zUR0K%H0`#yV%Xb5)fK77sG);1bl#1so=uGVO2cqzU=12sf#gx3)q;M2{fWs{1stIa z98b|BC}h{|z~MV^jKj^nD!ecP}uV4grsnEO!H{6$dof36k2=%SOx!j=6T{t z5Ht#1F#dvAb8(Cm-#;zG+yS|a6k68a^XGp>W)EvS+kxe{47MIhlvEpgm&S}AM4e-{ zWsUX-W&xIHtfTj^VLFg%z=9ueUMmb-TS*rzMtD^C#?2l2OCAoOt@v`d19^ss%6bL` zIiu|z#F$CaHL5-_ujNm!$BdKZ)rCmlT2`d`pSq%OpA%7`fYoU>_)N z&+;wWT7}77K#G}MtQ5IC4J&7LfR5kxpXBB=bnb@TS!IeHGO41hBr7M!P-c%l04|aS zn`YXtcNCg-XKlZJXRZAfhOk(7(y_~_#|lqfv8AAXRpfjW4 zqf~4X8;Wj5Xc&;C(xN{au}nptX{dmQ56>`g(%jcV)6&&JHA32ZO+z2a7n|tDmmslb zz&|S~Y9t7P&a@{6UXhoEdo+9mPBAN;?IlDI;$p+MBHcIow^bW9Y>*Wx$^u$Undis% zfOt;wa-jR`@OVT1CfC<_TUF)gCF=e=cN7;dx^T#Ef&>#*R| zsaM|kFwN4vdt2@GO|}vOm*5AxK^@fB&&!6ArHafB*hYZGPl}6j1cL@?_n<+Mm(n?3 zK$)<2E=G4v`S_k-CH&&^X*AI9?`dzLQl&`aX7~wcEmRdhaYl3m#piQg99R^) zL@HB$vK17oGSG;|#+W)u5o{i?16!ee3eGvR`g(0sxt|PwR?yC2t$HJ?#k#A=je8c3 zMTja)F@x_f*Cd90bjf#g1rrcGBEs7XUy3lE+vb>0`@v1c^Cs^`Q~3vF9Ahh6&+EqJ zHE?@3m90u2G=`IxpEx5z2HMXr9)pvxu4JateRAh~Ci6*vO(=i8h7+v7R|%Fv_3-cY zB`eS+q?(0!*ZmlAXyJv-g@_14(AcfobyJX?SA8^FT0+3PrDSgt)N#6fyI3D`398=7 ztXZ=pom{jaY*M$OLmMcz4i3w2w^yYYe|q6|=zfrKNgGEIlb!q528;W>^si6M-SiVJ z2Dgq!-Z6d`P@QR?UU6?f2Ft`5LDu;9*7?!fit}Ef5HtDoE3$aZAmN@pxT>W=$IMFK z+kA|ypVnLDNpK(juWT`2r_eI|=%bwe2kO=WNHfOX>eF&o1t_2!(O(4`bBu8?dpQX} zY51b#))y$>W|{WSV#Arsr6z*Wf4`EeN-o24!bVwKQevc_8oC&b-p5U&_Nw*qbv2q1 ztN$;F3sz_G0oFm6`2HBQNLj_p%snpYYMirbD~iYpy~O(a)T2- zB>uH^+tC#RliW3$r1OR_%s9bt4Ar73&=4K-I7UYz#NYQKDP?y40E+hfwfh&X@69m8 zg8no6%EDQvLV!+HBAHQ8hUaN3MdV94abeU*U*Fm$Cd-b#n7eP^zIzv*)={ZiE;XFS z4stHYG_{>O?PWwLP)jblI-}xD`%KQ9UX)!(htX}oF=ie5b|8bl(}-vR8x#z%v=b#P zMUAQaYkUC|OlcR7yPd2kn?vsgh`j~?LKU2O%+Yl`>4F9&+eu)qR0zAaG*V2e*ak~6M!am(y@sQuB!YGq3lV1DnPmvYpnHx zTZfuc1Ut-L%=?omYUp{gu+1)f`CAwBZ|&5vl`9M$gJixIV3r?|VhIq)PCcu6hSzheoy>aIN&yzo!X3Hn0G7}(>zcVRhjdS4*0 zX7A@C-?9SB$>0c%PybMsYZ-?A&B4j3FMr*Eoxhn;^u6-t^2h80uEd7QtI|t0HL^{{ z@f+{AnP>3zWy2Nng$%hTNn21fuyO&t@x-j#4xzMet4H*|vhcU&EWk#F@=fwYD4!ls zwXffP_3Bk>y`($#*V;;(kBS1u@3%(zj1cby7%nLDR$&b${eHNma-ZV1Ls#Vo(3u$( zokmOlAuc!$f8-=G(E@U*;%`A|< zSf^b<`HzdO8b^cxKtVwVm$ZRxyRa}CVF1^-LMcX0bFt&j z)RzmgXIx}ci5yD|?681HZEr%6({|OnOt`{`>_+^w3mRMM!ziPeE=BU=b()-Hdyqiz-&@pP)S{Y;S34 z3AU%k29T}FCyg+moA;9R}nFLVGd24=(Q!Z zfPcPoVnJPcxgiBl>#DBYK8FmEL$OEU?W1jxS6l%C5~K~zHjSfLMal$~s)-_+4dGka zCE`XyV{*u+sQoj#Xkb85EZ#gaWpin#qtk9zS9m9%Qi*6L_|7P+nG;c4pr4Pn9-Lvq zu>LatI=fk#QiwL;KWKkX^Q-E=UHr$LJCWFQzzo%3-~#HmX*ki}Uw$2KXB8~DJ3ElU zC}FI0V~P80&vC)*JbUykq~?lq;U^%)&=(>EIAgdaK0wOU#U@`*_(g)!C9htzV|anG z^|`S5v4j{&%QxciBqO>UdTF1NMF<284W<~cq)BCU#ULh71PGK^g$o;6G9X+Ap(qMz zfT!U=<>X>mM@^VpgOmK(v)cE|yWQi2lLshWEB1eJkcXbUhdyf@1wPTUwMDin_cUx$ zb%7 literal 0 HcmV?d00001 diff --git a/docs/figures/finetune_func_cm_lora.png b/docs/figures/finetune_func_cm_lora.png new file mode 100644 index 0000000000000000000000000000000000000000..753c479d0ce0cc6de25fed494fb31397440ac2c2 GIT binary patch literal 61136 zcmdSBby$|$*Dd@YAP7iEiiC)wG$=@elqeF?-JmGl-5}DUfS?Eh(jg!z-3mwu(%s#i zXWo17^S-}xzW1E##P`Q}x%S>}iHGN2Yt1>w7<0@O^i)pjG7cpU3Wd7-_>qJn3WeT; zLS5Lvz6hVG-s*3LKltn=)$Nt6p4mGY*qWeZ4eYJWt?bQT7+!ZYv9)_)WqFT{myMI< zx~aXrwH-e@yT!kK1DloYbN1gk#Def5m#iOY*r8Aa2FSl?86xQ~P^dMb#}cB-&haaw zPU<9T(H%6Vfi6(l$%-kFE8qQ1ey4T#4Zo#+ivq*%I zx``JUqAM(#%pW&&hB04X!D*kWu9+W{Dz|$+w=%g__`;)~@|2yB9;a~1?d|$J~{F0KA(xaPK zR)&hRpO(^IyM`Mi;R`P zFrZ!_uHb@(hW6&sO~b&*^dCPS!AnUze8^O3X*vB_x5|}5#M6Be^N}*PIteimBOL?7 zML#TphXFXGpVQN^h>3}vU0mo`Snx74GbO*T3ws<*Y|XaCa+{I$ebtK?DmEiuJPUr8 zm`Ho`$|V^Y8QWjetHY&qk&muNKFZgxyAV)%KJ$|p0|R4XswpIZF!!a~`Rqf>2-n@E zAZlHF+U5$6<2|ajadLciy=oG=@16V0Uu&i&8v-}>_um&+u~T#;2pta&d+C)`dYQg_ znQXuKd*OE~Zk}FE!n~{7-inXJ4WHax)-Q@LeD?Rxia!h+jaPdlp1KRSyg%Y+U)pb( zx@%C6akA4R?|ihAIgmv7(=>=0MWpa}yI;?1@}pI?PhK9I>;BqhCnqOkW8;a&AYuw0 zOR7BGs$i9DRnni;u<|xrGmm8w1U1$g91Lu&tslI6$y8Ka9AbQY*!}yrn6xy`*}*h{ z{pyg4+e%MT45@OrC!lz&luQrzWb9ht6sl* z_ljDcwoh@fL-^S>9!vj}6naA=Bb$Q_11_tv%G%@KfBr~1J3EK(E&a((kqjYQ7^@Pn zcW@9{KD;A7*zfZAY1Dc|Q>u6%J_jdfU7<-EAqmM%M_vikUt2#jR^^8L02LLLHN4`$ zz`&1~%h&GXIjxP<>3N;17$19WPGO>I+uL(~?7vXtgNiufSsE==qi2D z!{C0p+h0!em<5NCk&%Og!#g}&u3+Wso!%S`;&IR8+yMiU*uX$6Hr>kCp9GwemD1%5 z1C{QuX_t8~ESUQH`;#12Q{Fd!4bS|;)6vGJ?hT!IzUAn5Dkar-@7@)-?OS4CVHy4U zz&%iAlOiTr5EIjPyCo(qEp1BPTljJG7;SWFQj!n+wEpOK2W&jNljQ1?zbM{Ma~XV%w2TnNoa-LDPUp!lalD#+uOU6!~>~%?vAgo zcYG4q*xvSwi@Pc#E8AUarT68_mzhrk5j!h`#BeGk4GoiXR@>w5l|fo?-CK!E@e&ie z&>T)3lBJSe|KkTX0|SHU=y&eqloS(>8}#%F>2lPl2M`b@J$U0H@{lIVDk`Q&%I!>h zGl=l;@Qhs!58|gFq94f1v$t&TZ*KaOm-8hEx@`TNe;*$o%4yVGJ2po1{rh)o*acrq z-0RrB+9E*`7reZ@AQ$1Tt*s%e_3@)tlC1o%W$nd?mY!qI0*Lw?&0^{M3Cq2ie)pbt z%&4D3Tr3O~GfPJ^VS0FYl$Dj~Z_WI~49% zYCU!qh}=Cq;7MQqPJLq3o1wsWrptH~Z`YG7i91r`bsq9@O70dpQhzW{Hd|;jo+e!o z6cWPi^{Vwoi@k3_`LZu32vR~4oK|dn{D=E%qjo!0cJuh(?H7H?ST(QTyooV4HwOo> z;ZL?28tN0jLw&Mj2&(eql?u;OSN~V9m`KjyE8VaK5E=A$?qJi=(V4>Ws-`u@rxC=a zrKJ`0I-`P8Lm}kKb#^#U3zcKmmF`oXPQ?@y1>~3Z=jlLU@~N$@z0h*#xwHc7YHacG zzJ=IqYqSDv1J&vJSXBsYR+7bViF%ne9h{uNw)gBQ@TLnd{Q2`o<-Uql*Jn}n8c!jp zKkRB5jV&!H4$FNT8ygQyP49f*dhyoG-N3-$j!G8E(axgZ%8EIvwl$i0bAQ~+=;zy8 z`NhS*w#zU1dhe|c8+9j3n*PZmC1caZCMPEky~cx|A{EimeU4NCe)~J$oz}x$w&&w`{2Ms`t8PY>A_%%Q-vG=?;pIA6$iDtH^M}woHNbs)S#ds3K z;u8@;5W}M33)y&n;p3-IVSTw;4G@RhbHDr`Go75CB1@>6tC{4uIutA5#1jq#Nz63_g;Pk$|Y(gp$~Lq*LvFl~t=Gswx8ZgdC!!(<$CdrhC8Td1pdhPY?0& z{(1}KNKUgpmM^KPNdOpZkL6@#CE>W_d!2h893LA&HsSR=an2)eVp_24c#^AGjQU(F z$)ylU{Q7mq1Kk==Rumz>{le4FXXNa9NIzlQKPu`EHq~o_TU1_;Oykkc6g;HJz`>F;tl;PcnDnD>Obt7MUPO|E1frQ zbe4^;BL%7OJ^=wi5?~ZDclXL1R(^iXuC6XZfES;PyiSjH-wi*5M1uL`Nt@vQ;p8?4 z7Z(YJC?SO34KA}j8A1%v9~~WJu+RYz9@=(qVqzGKpFf{>MiQ{t)2DC#O8tBZ!){^M zc*s!L%&Iv*95i5iW{39H$JtSY&pP56N2=V^AfsJ@M6vA=Ug5a&LZ}eAc-=& z`S6^}-k;$N^^;Lk!;-)$+xmH0*{Nh1K`#|Pb6AN%H}C9P!}{dO69{e5zG|qoXxR##U8H1z-Vk*96QYNU(~MVKa^$Y%DD)995x1n|gr zhbNYL#JA;0xFBC*5zx4Al@AUMUV;GZ=&pYDixSP-dagZAuzz=Vm(_Qe?8AbN$1U}fDM3Q29TtDzkU0L{9>>3Q#H$q&IBRiqM{=I zM6WZAzPZIkjfwrzeZJF!O;5opFH1;`?{by8_RBke?WgO`LyZ?p6!x$!@;cFUPLzkk zK_?`n(?>%_#(*MZ)2n8|uCK7sDKM0*YiW_P3{C&?1$(ybJ-KF)DV^CiyiLFiGtG0w zX8jnlva;p-iCg!K0iU*k}w7ms$k?rp65q4=>#B=<0e3Yd&mM^YiG|?c2Bj zQEb<*1q;rPG}pQD^YhmO8rt$0lW6jh0)BzB01Jsi85vDq*d71&oB%b$TgdsWodWfP z(;*kiZDD43ZAA!Od0E-S+GvGz%Splh{0&I46}4&~!a_nCnnK7Y=jX9-ad9==ieLWe zf>JWLebNQVl6~I}Ne{HN>}>|32mpzUj06nLh)*jVnl7I>$H)B?YBVL~`q>wg=g;Yd zg>~<9pFk$W97_j0c(k{w^|>0Nx^be;&xAbScTW#)oA z5Uqs>n;rVdV}*bF_U*yWKUpe%K|zmwTADtK`dLrZqLoVlRKqP=Yx(Z5%;40xdFt#z z+ZIC3ZoM+VxwG6)0CnS?mRjWYMG-;GlT5Af#pUI;vqS(d01WCOdH7zRs1Ly9J0hUq zc(CUMSPZ~NEiCcchzz9s*toc#vrfx>xuyg8bnU->&4vfxw;H2@1IO#QG63}$a(9KzCJR@JoU@{UkjjM4n|Gw z!;9fvzI@q!ZA7E5FGY%AWwe4EsyWkBcR&by&8nXBvgCj#?6!4cVa1r8WPKu9PGC<= zdSa%hr%ehNZr@G;NC9t#;)^0<;kR!ENWwchYOT7jt)nwJJ)L5$9b750Huwz-YM`Xw z#mmMP4IUcO4Y!#6*Fy8iwV>4un=gaoFs zq2UFT^YNaA&(U~|7cC_vrM07Dj9F<($t%F2YvVQS{bx2rXqCrZPrC>*FuQZLc>Dtc zjSx)1YjfiRx7jPXPy9(w-m?`Z-z}X-t7IkeZ{ly@j=oWy#4OlUCb87jZ1h^6Zc z^%<}pj*hY!0MLOdHy(sFz-u5`_At-x{LEuuXlU-hwh5p?$%%NP02`BPPDWSLJKhs6-r$=UHb-`3_Pm#&ad6_wiPeMmZ= zKIw$}j`sE{Te_86jW?%0dB0Wpavzf2wui8o@VwOg9PH4e$B%z*7vGMS-rSfJFDozq zIRsDk>ec0IHy9Z&2{^4qk=D=xnnYjf%MCd=a2`}XfdqP;o*rGR)bf`C_UuC-mXif4 zZ#K2N+-saS#ny@~jg2wr>FEQe6u8FV6aZjCtE}jFpbOP2Y-h**OIlj8VN)<)OhsAQ z%+c&r`kk#aXLx+~-9Ku4Ln7{-Hcn10us-fFE#x$Qek%c@8(^zHq_ZhVDcIQ9KfbN641VKuSW-p;^4_y{Z@{fwTwENG z;sVyHjwT~pe9NpSeOJ(rAT4%77zFgOzpnpbqI>hopx_a$P6&%g+vS~gC%L_oHBKfRxzwu0#*r(S*+F?yBjErXEFwkuEHMMf$lkY_8#+*pqY$o<^OClVAClwVlL z)ZXRqyJX}uPh+EcFl{P}ZXxRgChIrDkwabv6b;XMj)ZLFt)g%W;^f^${D~4~&nGe-jn;Wrt`VuyvbGS@vgy+o7ZT3F^5) z3MmxhVP9t_;ReW-qh4^Kwjxn#1o7bPPg{Yb<1sU zwGfR&^s_G5okanAgzynxv!aiTb(Z5KioDPqK}#SmE{;dRfr5J9Rv-w{7ElA!n5-50 zn`xYGTyi!H6rxoGkiCERju;6v$YVc;rlzKBwf}Gd1Zw%dUbEP_{ODC!*kwUM!FUpakn+d~# zhLWdy!`yh@`=KSG(<0Z(k>%;-F{KAd=J7cdu|@ngkpKl*J$*w{5Z^9(yZS?^_J{U%X5? zp_^7rpQUltzAAJ|U_YAdnG&0&9DFiD)Jj<5N&4d?DO^ z9;JTyq7Pq?7Mhpmb6TT9ksQe3OGt(xponB*X2u_>5s+)G9ULS9?@X?(g@=aXoosiC z419OUUZIJFiXa7`F8}2pCJ4@C;Fkx0dJ15Np}x+!RM48(ua8j^Gsrvu%+Be4U=3P~ z{^8avWSX}l0cjvvAP@vOJ=WINwNM=j%m?od3=aO(@t-wL_Pt>_U!)gYOR{^JsrPV!kucs41A#+RjdCA=#8x6Zj&CR^E`541s3f z(W@2&Ah8LX&FCQidw-tHtI`r6*CZ@3(7Y*Nj);&5ye-7|!&aHi^v3aeO$zJ}Byw@U zLT>EesUUgQkM+<#n!o3(6@~6jTRwIm&bNg{kRvK|!iqtVy2&Si!jzJd0-Z_Lb+zT4%aZyyt4OafT7bMwQl{(f@gokFsib8Sn4e1fZzrA!3mY9`@1 zuv1JPo>Fa~yN*!L=y`d0v0=|h?%umMdjwjXG{TSN5(E!_&Tjx?FaS}@H#ax;Gf4W% zIqC|rB3_>1!(?cLuzxW|r+ez|a(F7I)`JxC8N2n~Q-PoWIbZ*DUlw07^j#fW{ z=YXOlhEQ#YB2lP=AYT%t$$iSQhpOJ$VzdldSPaw@OsE%iP+Tc^tv|2CsL+OwS2nBY z)@+aHMp%tklOpTODh%rPnRC7Oc)p(r?CaPIR zfX;*CdmS6Xi5xfs0_5syYac)?25oeTod2*`K4tYUu^j!dd{$Leg$Z4iy3S5{)ODwj z^axB04B)E^3k!nBGy9sqmSbcvT@s1?J3F0pzuJis`oDQYi0GZZd3kvgv$MD2h1~9U zYTfmS@2ni7W!EmdZoky~7_c%4==_+Vfg)$V9(H=>h+4>1IYT8o23aQMjHjaNKvBd0 zrk{7PeF=$)=>bGb?tK6;J8uJ&p*eM^7RO?6jG>aF|Ftq8jw3j*TB;r=PTyE%oCxSbAs>-LQ<0!$A|L-7y);cj ztOv>iJ7xzV)M7zO^A$9PflqIo3XMEjA(W2ak;xpOi+c2Z=mh`c>uQ#WD~e6{NsR^H zy2u6muRQ*WB4s~sz4Fg5H&l@Mr_mQ8$@$mkh%#Bp{(Aco5?@M?h6(Z!IrRVDH@<9# zngA*Yi}g9cN2JBq)zjk_7$_ClB>C4fDQg!s;(IrKGit#%H8UeGJKtICL3CxH;m}6N zz@}&Bt9;3Cpu9epp^-{uE*+P=oS||+zx7s|_WhO#hn+t=WuN`4n3#)}BObvtMh%-A zi_ECnw?9X_cbLfsqV7s6zT}sa9B+uqA1xz2mI+N)X32WbExJEsGO5^%t@@Dvz-Hee z{`KQG%RQansD{el`$Q%%u!(qv1Oysx+~Jfhaes2Ina}y>o!0Z&O?7j@?K1TQzpSCk z7xaC38ezcfKfirqB6DKHl%<-@;Hb{Ddid?RIwhH}S8_O=5bI^fy;%-!>-$+{p|<(G*7ccgRE zv+#BuY!3U3Lh;7lE^lo)s`Bo%nT~wlClN+(c$#j;;p;UPShORfk%-$*YxuPGqO<*n zpgnOfUx^!aiEXb$#l;o1V0UzlB=le|kt9sbk7rIX*Jd$8dmU7#U{bI+|WCeMsS zZ}Tjew$9%aToQIyJ7IP+uOU>Un#|7eaNF0ftJKG%;m7+}y5=A9#02l^2<_(PreR=O zA!Oe$A5WpA)YO3d)a-nhmP86p*9%be8u4XQZE_d`U6iQo(x~QEd)V{@1HTG5ivMVJ zh%Gn81c8skWQXueYFg_NZ$+75@0SNT1NrFp#9~AVF=VKPFd!w|c$nu}9aYertRCjU z>oEjH=iBqnhc%w3`sU`Lz}*Z0s1;UU3Bs7~_$2=HDIv6qFI>3rZ_S&euU`p3a>S$L z!hZbtapCUC$;rg>atP8(0ZE08jSa9v04x#GV1Y^?rl&^>ZP7ZY;~)iI?akA<2AT)b z_|mO(R)}X*7#LURCoEf$Kk=MV%xgnG9X2=*%8sQfxz^1?J2{r}{$P2*q|$j;FWhsG zC?Es0d&2kLAv`p68|fnYRz_%8TC>&8cP`1bFx)sv#Nze5P1ip=-kaZa)I(r%g8|J+ zFHOI%dN`l!w* zj;jM~j;2gzqaUoJN=oXAJ`Y-FYhE^K8%jz@Ctq1<3;J)*>2UflY>iiB7W}nImG_u< zEA)3FM4dNT@SQAm-|fB_t;QV@dYKd|s{#9wyzgz`ym`WyAmVYua3*X1$yGC!;KvNm z%K?Jk47`#9;1tqW;ISIRKs7cs)&2T)*{C^GR9l-0=^W0?7#|!Q)Q*f$J@1IG0~BJj zF(C?)dYi>QXzpb$+YIM{_1^) znQSa!_SSoKW*S-Ngf*~bBcOs;V5Sq`gU zqmHPCN=7!m!_hh%dg31K_vxu=Q?&`BcZ`(CR8w<9?r}x$-3sGQP?EU|k5LB?kA?#D zUkfcT>zN-!h-^4CM5(E%3G%vc@ooi=A85x{0h_6pT2h0^hlT>~C<%%?htryN6r<8b z6v)}HH#V%Hu%l1_qR~(wt^@$=M(H=cCR*rD!2&3T!OqT(dH`h?N{NEDd`(w$35Rb% z5=}GcakYtOjZyhJri&_=AH^@$KJPM&=XH|mbL!%`%xU(8JrvL0W_wJ@UC5a6?zdh$ z*YMQ33Y#q^NzW_~FUMPWm^gUOf zI>|ekVgk9>?;qXFFDYq8#IzM?i8ug5(NQHt_kRDL90X3wu}Vhf;B3`gCg#`ML*s!S zBId`vXK2@jgs2x67i}$kxvpKg{7XJhjf#tFOt$gh-qzh1bHNL!GW%uD>MUI}^a`#w z%)vbCLVn`uR<%Q|81!yDEw9Q?yNcGuzC1S$y5jbDWBBkqjeXQ3&Ns)LvH8AJeHQV4 zWE|^jimTB#6*E!&OqLY={GO!MP zez@xDOnQE0nCGkbM$x8vKFbCE@86qf?~pReB{IZC)Eq(-ypa|;r}{+@GFQ2GJMxjO zMVr-mU0oeCf}`Dxg#`8oImutW|N8b#TwK_)(=7X${q~yDfY!&h;;P5e;_ZIqulZ^H z5{w!q6|3?1xpH;mC$mBi&S~lH^Sn8~(NzER+NGZ^dKs%jJ*8d0d%cT;bQ3(Pjpv@t zb@leP+ZZ6Kqq>Gh5J~jXHzt#SEjc+`i||?I35|GHy`oj6f7wS>>o#)C>G5|hgj8hs z*}qL}S$MeaKGbl-Ds$_tkGMxR(e3AUFn?e`bVEf}@3*>rnOG<& zC;*E04heZu&!yHB%6yykZ)7EJ5^3{j{}jm?$oHYq;*5re~*rjw0pK}}tzYSXZ& zblb!NG(YG*NE9z|ON7c4t7PjE@1J`~x403kwbTzatHygvqy8n{XXOboSoBCj-?iWy zIW91%567_nkoMkK6V!B_e?n=tbeQ=OM-l{i4oGc)fv})=4Lu`F&x1)U_x(|)!o@C0 zS?F?nrW?i54=u z6WXnWR=@9Vo>*Kg(RKD6(;J-b&gN>YdMxhMY5Q8fwKhm#y*kML8|On(Gx~V114_oY zLF-aZpPEv7d~GkEFZR2FG-1}Rvb|bSV$~F(gx>VugMb*;_^=eLeZ!@drwi98ZqS|GLe+Nt!2bT- z^|jj;b_>C(RM_`jgO|_3-*+Y&EH`s^b^nq4baYuVT({Vr(I`xklkrz=Eeh$}z9yo7 z3JNRm-_KyhfR=vJKoaT44=Pj}P6yb7o!iZh7g0Iqrz96;$GyiRwCf4J-8a&%iNU@) zj#;`UA{AjXc`!d!5kdg*TbKQc_{7b%VWbh&IXynrcqm{xMSjC+IE_8*$yIumwWIOF zX%Xl5Sg4h%BfPrfb^V^Wmad+@#sM-pj^BUkXIrc|99vY3PtLU`+szq2sIp)NkbN` zO6+VkRzC|3flwMAqTFiB+fdxYjAZ%RKTgHogKCpl^1NRaRogg;J~JdGha(YkpBhM* zU-tVBqy4_HN3O>gpOM8_f7K<%jrkf7R~|q?mz~3IlJgitbb|E7LP)Y+gUGqu&GZf@ zp+@{9i6ghN_rP0t7vq*Z*k70|zT7syedDN5Q2G%m@RpViG5F>vKV39@^zE-Kf3cGP zG*snpiuJ!ir)~ntHpCKDj{RNO82)*N&wjcbp>)5P-jfMnAXJe!XDBM=>nAT-8pi)4 z)8${5&HrC*oc}kkxcCiZMcQ^FubNHc=;~`>Yw8cV2Z?UZRIK+?VgY(*9#PZ6FHTz z{nM2!f% z!5k4wz=Kin#Fy^rJ3e9=I?>q@1|?iI5+ zSu#$|KTQR|R%C73<7{JjnmC`g8jO*3ySb7MwpYjeEVvM&%nG-nB}v4)C)IA^|n zy9WY^LY_8X`v-1Zl=b13375qX$xF~**#I&Y*ipnDJU~I4V55z_=CgLW zE#g@N92p7CW6;Oz06u~)7}KQrHdn9W;-~NS0ah&?*0zq#QHwr6&>QlohanG}n3}$< zZyZtFyVW zxlDQg)%k{k=}=|BK$81?vL?MUXjB+<#Ph}SJ1~Ky_X?560n9&=lY0e_6xvcYK&8w_ zzmtOH!oQ}bX5>gJoJtgS68cfTKqVlwEp)D>w~JKtv_C zfi*>1fVCAV?VyJ?fsl%rLJgrob#S!&b#tMco=V7-1Wt1k0Ay?M8Qo};S7>YhQM+rV zrtq~oJx5kt9DR4mWlC`elZal4m3^9qPT!mT8UAAMcJS||hOUld)0!eWi9;O+TH4f{E2kPqT z;L<^XHVAn3%$Nn-021IbnH>Tci!|xLDw6^&F+_|9R2zgCAK=-5%ObChtSOkovas(e z@B9tJ?*j&(9b)}AAkKj%`#L@==-nbN>G;jfcRvT5#vgSEt4kfzHrp)W~bgQv1rr{2Hyctqm7~0tjIg zX_KX;CE&%VT`;cIL3frkJQYl3d_$0A49rjT3Y5}hu7dAc4BBXUt6)xcvE=zTFrLN+ z0Yw@f**K~BEn(%nB)r2qLmBg4_4!5e6nycIHByLl$7C2Ue2UrA-+#Vh0a{brDFXhY zrAfy+0-N^JOL7;^eEs|kz*4;*E5f5bc@P|29>)jPA3Nt3i}4dVHSS78~NcxHBVmA@PbS`i$w5K?tf7*{yir&CLiFcg66rfygUtY$QM)o z{re4meoAPP7M5gW+!-vIeTz8NoKKI6M~p`N9~%bVO36(@2NJ;RvGgTJmMMB_XYpNz zYSk-Xn0Y(BB$e)a6by5!`PmZ}WLa59zT3-{oE%PLIqz;=rP3ae3{~8<{&y+mU*5=K zj~<7RkdO>?0ilN-7#@xflKI9|6RviZD`}PMZasj{?sR!t#3v8tf`iFG+Ko}C@toK618TYFZAn9zyM82o${R&PV4=j*0h zqap=G+=I?qLIJhF`tBy`@aU07e4W|<@I41VdHfjZ(He`0c%ESMSZW7mW->xflnj-z zLDDMpoB}`vNCVdcH0o->5QB6e!LEvU@QnzjKOq|-mRR4Fc9}BgUdef%6v=SS z|9H=XD_m!BF77R}9`hNGRVYymJ1o4u`fcttX^i5(K?mj4M>dbG7aoq0-%F|F&eqmk zA%q(JNH+haQh2rFnFrE6{_EpEi{Sfzi=tdkm{e8>AIT1USPY@Qka;OcVg*~8!IMc@ z`(cbS21A-Q53tti<93Gd-NPIR&c#@%)Eq zIqY2JqB1!@FGok|-l~*n41Ow+C=}M)YaCVVB=KJNbaJpP&e?~YFoJ~Kr@dVUR4_z8 zgC#`)(DDZjv7w~}3mTD4iX>i0YaZoYT+9sxMKV0ic(~MwfUF(2c_OJH-(^{@tW3H2 zP2DNtMFgHT*&1MHI-GfpEKfTSB&O>E%((BQE{kfuWI|m(o5gLcarZ%McxZHZWI@5r zhy7e$lFnZ4^N+}o5Eo`DNMRW>Q@)SirH4>U2h~?H26_>Yv=pS3@mT2^eb$qQzB8Tq zB9r;p=Iw`}2`FgO6^JXnaQE;W`m$t;J3YU1(qgMn|oTRb7Kd zD54EBDQ9>?tI*iQHf@8P0(KpsI3TtXa9)?q0VrFIrDfmai@bO~L`e>OAdO|`1(m1N95D2FcV$vdm zZr3#S4jflwJ$#6R>l_?}-QC@VWl-i(C@@7)K;s>;0l~5%&>FsWI4Ab!e+G7MwX2um zu@2dLi(@`l&RFp67P&at^o@6<+T}88LcQ*)(zAex?2BOrN_{jm#Y@SZ*S+Gy`GZ#o zbOV=O!FK*PFgNN>`$yY@{RcKwX2Dve3{Z(FsC2^~OTKz_lkMrgVqoJkFyWMh^Ta*_3j^ek}oYaShl9rv&^-YaK zW3~DvLo4-rtM8o6eY7H^r9~XmKZdScxq|dZcfg|xBDfSdO*jo3v9&84FEgvY6= z_{kH*L$Lwo3#6fqm^!PE*91<$VuA9`&*uQE87(>%z62=wfW-B!tRhy2$E)&T4h%_3 z0C#_ul!KiGBJVmA6Aoe*w;P9sd;|EEtYQ8I`Ekg3%p0Z9I&nZgJ25xM5Rwh18PwP* zTY`vJc$M4!6m(`YG`=w)E;V347c8_$|HmSH-ptHRly^bU;_YJ!3QAF>kmuf&EtN!> z_s@hrPuZR<5Fs*NV*&kSPb2QQ%<2r0nGDpkC(N@u!exorM{s7os>zrmO$jh8(bxmrQ>5&i*X-5h`~e=1ROAcSCG5_ zwL+w!q3-B>=K>f1Y==vpdch6YXe@9VL;ql=+zJM#zzcj8I{E6=?tI_}Du4+IccY6a zn2Lh2CQ;~uk-e|37Di@+5DOISg(U2e0)SYs9-y8*do}>mH$Ek~!T%j?*xZ_Z0{XoU z;Cg2APO0cD8ta^ic(cU*psyQdjh&{Wc}kp8>EB~{Zv26audGcY;`*9eizh{!4~s?j z>>j^Y4>jHPv+2%pCqyWmT2|tOSIU0R=AB)EMb&z~Uv4zAQ~sm1wcd&P!^Fp!-F){+ z78yJoA6bn*iK(>>m?(IT03UQL!@)ums1HqG%0b3Oz^Zs105?d)wo# zorT7vH%leo%4VtmEBkQ`Ku~z0u;qRcD@_SRl79Qvw(@FfTG|Z;28PaE@SR5{B(U`} zm@aJzE?jvzSg$`)3%a*oMpETa0|{zZ{lKfYOCjwqPT3Y^w;VN{pQbQHK3(rE(|6q4Sd-}Kjn8)jit zQoG$jELh4gMBkXT!8D;$)t4_fVRUom=tRUJdSzllALC-H9@gKIPj?KZI<91tb~C zIH=E%JCO!t{3pA_w+H1H@hH4$gj{Q3ybk8&Hnz5yx?JI4Ep#QJSD)@_q1MwlVR!^F zA4Tp$=@E5wEE#bHevW#RqC*5PFP8E!421$m7vjo-Au2Rfs&sTc%w9ys$FndOid63^ zTH5@95mhi|BFqLTFfLWwm&floTz5>%?~5uqgkLdRUO<|Ra*6zm6#@RrOihbDx4Wxa zN^eM|Q^G96mZY8a)__5~#ua?C;X=NnFE?)+v@Rqe6}zZBscTXOH5ZoQKRJ%h#P%nv zzwvZHy}Sf#-^jqih*|EzEtFp3Au$G;2zx?^((Q2lx#UEJ=7(o?Qm(F)s0Yb(k!Jl} zT|aC@jZ}cwo+I%FCbub=^DPu3MjsSoPjDyI)YJL3$a5n%Wa#oKWTUe0-wk1U&Utq!dSyyq&tsPq4$8)Kgb0Fvc|DGbR+fI#Hamcm zU%S$o2-=K*i64Rthd3g^Uug(-1Z40NadXz39pNG-WjojJd6pfSzh5vp8jP+V?KJh$ z-md-ii+W8Z+o0j7_>rt^fR$9j@Fi56*faXA30)an{D5j7x+_!TpDK(@jWKj|gpl># z%Af<=U_;r9Z*Shlhepj9qr4Z-YX()M+O;OZ4gUY7v`oScY>&3q6{S41Qv{@}`d04{0RxMpuObVOuA zty^jp+0Dx#6CZs+p`j80-Zqy0FvHji9VaIdltRS02t;FcPY8w?+7pDXf!o$UEKJsC zO7_vC2jIZ4Ug)BObgK_$PUnpYG-MR6BH%YzC%QmxkdTt1hxQPRJ0XLG&)Q>sOxj{5 z;V>+WR`39-n}Ev&V4@91KU-Jl8|KNbT?+vhsu4`?VPC%7gv<`Dtb~GQ)@r$nQCWf6 zgBU;$HjQAK4G*AA3y7-5Jy|_HJ>4*ULB(&64HyxM>}BwGAZ|aHslN*0*8*m6eG7}Q zhhda-yu1`J^^18To29FOhpMq0xn6O#@`;I>%z+IK80^^`)$c~=rOu6KzAkz(5ovau zL5tp_M8MPE>E@AduO@@yhtaeJ;=gKBZLY1ct|QnuTK% zMdRI~v%2#5%%k!1k{5yUs*=HD3bU&kSxzJ?2L<%%CC)@N!es=*g^mkw7!Dje&c*VY z9;<2X=Iv)MeeHDXd?yKS``w4~l5-9^)Gy6qER>+j4r*DIwRxABJTm6Pg=<=F_6vK| zABxPi;*;hkUVc1hOlApILHhigAo?IfCx(Grl24nG)&B%(ki2tMC{B*8*3Qx-$>s;v z9nVD8bjt_+lO@-lTy~TBkwW`o&kfhpBW}ba49ac6^}kyiSI}ybi4+3ysk;hIp17Z% z?wfWe2aFUV^P)d~-05$D>M0rg_stXwZS{*iOiNHlAIbjmkj@6%DhmrsFkkXpzn*l` zmc<@rZC;NL!MN%2%z=qG$R}DScV0Bv92ME}dVI%V)>0K>r}>Bc#vhPg{?93pAhx%R z{8W=?h&%9IH0x{lV@BWPx+C_jj?7q$0`WVMOMDv0XX!UAWErmn4G={M@@on7rJRT5I?=CX~uxT9f|H(Fss6C4sT*()s6oVzC75-nr`sg z&Edg7z)eRA90jgtV59xJvs8qgqUceb`*E5%f{Bl2`&lO;NCX8A4KG7!dT~cJT_!8= zWjW2Z?g+XPeoPq;WA2U>v?Z+G%#5(iy{11!aQY@8LG=VJyX@D;`^gR=X&Fi>Os)>+ z7vHn%ToJe#U2HYXk7ubztL3R1WfXTETb+sn$8Wg5rAbLj^0nM9=)7|ygZC(eSKNwB zf1_yD*m&I+J`d1EQvmA&9fPLjgP|&$UhvQ#omrYs0!2guEUEc-$VB0)4Sjw6>wiBA zz<+fuE%6(XBnHY^R==9Z^XDfN2916OTcaL)BNM(MmBybZk=W8tU14yx<$#T&5x$bc zB10>H0mha>VV*n*di_d~m%W*RZ%D5Z7TTf@Ij-PezBFDhTx7LQ!xBp#S|=W8u-*5` z+9orU&hHqRja=zcmRngh<1MiwU!nSBr{lS>|Lf)X*)Aiz?I*IbqPNJ&Wj+Y%tUQ$_ z3FmG?)NE*GkjBRstaot^8aXsRED%@35m6xXfr+$akda_00Mh(rpg9=OnhK@jyW)MK zJd+^gmJF1^cKP|UXZ5YEI52{Mj(U`;2=`^AxF2qTRU4V6EC7H0g%-P=1%=*REpiwx z&Ij!uMj0ccqi<=gLCOb~0uHbSXd;6Ajx>w_Tkak@9~D6Z(%G4p-*E*S{J~(*CW4vB z3Ue3Bo^*Iy_&hv33rkDLeJ3zEV`~G@3z=zJ=>JL!1L{cQ4Q_?Vs_j%Fjnh)E{hbxH zVly?MFfcfr@@$nKc=_E6^XLB0a)(p2U2aD?Io7>?{1+!*8~M(#Onjz<9+&Q^7l z1YaJAwv+HnV@=A1+fQb)p8wA*cehx%F|)-lYDfjFZAZy%xvQ9)x3KyO_uyibE?mTZoXqzx3bF&;+HghA&3 z$>6+jzXt;5!OJv1R0?}?0PqKCBAN}?c!|J}OILxB6zYMsH3#J5K)AeR0g8_iQE-a9iu7|_^(QF*xaxyqmk+5J=$-)4M4Y_~-xgp?m2nLs+Jy;L7 zj(~H}J35*a8RlExX1NqjBkcdB)N0e=INUHWxe$7U^}wyoz&EGt%#*vj_)ZGttw*n# zY%(l|9lK8)&s1*Mx=`o5wNM>9nH8lL^!3fBPnVxOITY*o{rd`XQxYT7AG^5$G{G`A zVrn;npALsslSvs*%b5h1xS8gDNaO##&QR0)qNa6=DK9*0Z+>ecVl*|g=)846baCgD z44PmNbR1+&0V{B3ezdnsjDA0-_Canb0V`MDX>n0eSFxFz`N)CgnTM_IU1Wv`oaAUI z7{KuZ4qs&2eF1E@g~(twj1s)F5`+;hDG)bj!)09I#wZwrji3>FTMTnn@_F*`d6FUbk|w$8B%xUQYx*Nae1-Q#o%d(MA9>1 zaY&iVKUb@v`?KlUufwD~-WUJ$IcLI!$y@IG`S}sS@K68!$(GDmUdduMHlqC*&W?EDAU-++hsZv?$@ zCE@tm*=3VEneR-8a`?QRdb)wq2dxQJlk)NZd0MV%A){3&nrC4#z@k3PSDU_F3RVm8 z6xC|mHb2h;GM(dy2gUrCgEV{K=+~=7ncN zT!SF0`T?Gne$N&LSA#&SeEJ=^8Zxb`nxpOm_4wx}aWgQ&Bln`ffYTf4RMW1{2opmp z35;2NhIf}3`W%D|gjIpYh=KwoL936=w5il4w zQ^vnD57(67!QCx>SMNRh48MxkW$Px)q6V9^39tVM$;-10r=R=XTbb@2A_~oHkoTsR z9jm*3Pd!;PVkX0!9!RQ+6*ME-*}c5UbT(QpaPi?oQ1ASp&bO?2Y%G5uy|b=-Y4($j zwevZx^T~BGV!yfhZw|X#I*HY>N7EQE_uf$TlG|Lph73j=H&zE81J3bH@S$tv=lxl{ zxU^W;H6YQKFx*tkA|34$U2IX%sICazX0}_mt`g9kIPJ`APD=r4X)w%~au4RZGn zWGZBu51EQ&R7(4?y9^{)QAtUm)KUlJGD9$Jg9QJ^_Rq-}_iFEJR#DuR3$?+4@>8%9~aSFUpLTlauh5X3cS0K?6_K8*p3wc_I!p z*euE^%m4G6*?dMby_Li2^FymF|0bs6&AIrh0HOzc!9UlJr(G{h&A#DWSx9H ztHDmXZC^ChD!+YWsD|B^rtB(U#&E!lfG6QvJVpk2s*wJmAP%0=2fLKqL`iyy^Sc86IzD^!&ou$A>62SmoyR+m+Ui0S*Bx^d|)QFIXT$8QSLg5TtDKJ zoklst)Sa<>q_SOhgj7`wgwcp1<qC>sBHL`J%+g~SIbkfUv9NKUze3DM-A7* z)j_;>)5tJT1^I?>zT={W?dO~}I~?{edoaS5zfU{QBnckB!oW(l(Qz_Ab@1SPm?4aF;jyfLdM^IQqNlJeb=+TZ~OlEZQuK~t!Gt;`@XL0JkR6U zrvqw=PRK0Bi{W@3ISaNje|p>snk>?iet2xL0zf+vw0=F?@nRYA)FB!t1yu!!ltX|i znw?kU^3G$V>+wRr0@Mmxu0_yfl;2RU5E2~t5buR-KIoun4GLf~cl_~r3Bn#p(S-(^t~z#mxG3UN zyy4*nCJ&XR1^F=S**6+@BS{qQWa7$$42=)0Ffq;{0Z&Rw>RMPBEfVTb#Bqd+9Vvcw zeZ3^|f}q~D|N5R7#xB7!51M&7JG&1qIjd!$|K8)f7d|+e>wn?rHuKX&YNl7cB0OH) zmm~6v+MWya6bY#ZU9Z16*SJD-!jraMNc#t2S4rl zc1C9vp4x~<#h`s9t;YH7XWlb{Vxlsr@WkroCnqMt$(^2I8mGvNmw>!Z*q)bwprE(| z7zROx+PoF^bpl#}5L`xC%*d!h^`F_VEsH+g)YNqO%9TsN^->D&9ePv~Pjyp4{J_U~ za4^B5g?C9Q1jB87{J1yrafzc4>2|yEL?h^60Rb=GnWNw{=S0)-C>A{>3c$H3xB>2sDqZ z0+R9R(4W6rkOV#eO`zYlL(33|PAxiRo%;r?D9M+(PX9WQcj0I1j<4WYR=}?YE-mTa zYxJa`3+!A_z<1KUN6%J%dOes~M7v(f%#^%x<%%)hA7a3eutySr;d6yL;xZ#~R@{TMsf zsDAHLHf@Xj_dpBnb?w%bT$v@j@j*>_uB?sHHidUA9vxd$s>bXU(2kax2NGkV)J8JF zy-x@-hI-0t6{nC20>hB0K!UdWz?2?v%-c<_>R8~9eL)sCf!Kjw$%jNNrM8}oA1*DlWjFJ zHk?v;cJ8FmdB@x~uX7JSGs$2E^207>h%nvR zltl!Nz2XWc961Q1W80AmwN^v~Mc1UmbSb=;p944HL#?QY9T5~e8y#pjkq%nbx+VWA zB+VrsKOX8cu^ey7iANSiA!tsLQ&5P@jzr6O6i6aidN_YVG~R|5&@RY!#X6yDcjFWP zRvpW*b2)h!)MK`KtPD69e`Wti`H3}?R?=S_w|L);bh_p8mURyQfjh=)m#}JZWH1hG zVw$n7{Mc>v@fnxv$H(*ARzH2U=3t^)!bFyfq}!lcW=TbOaW@MTs2o*QGS!{vos{iL zgIOG~s}Vv80dq>wbR)9T0y&KHpsT=HP>RixB$mwP&9I9pAvO2ZCkSW~Ai^gi8ff+z zPMP;2Yt)#%0laoWvDq%c=YHp78qLOv+{aJj(p<$ zZgEICShv%qK3P0zdMk~!b>EqEgJrAVrz=AQ4c9O;(DV!`dxk)+rT}Qlke5hfL@iiA z0eZPpZ?wRQt#sQ3cPki)zC=?^y;6VRveU{Bin(*wdhG!-|*ULdEYRqG>b z#X;|qkTItb`LX{0xgl$g zajku@OQkYmv&y{;9Z(U(n!?Y|Z`|;CiQ}(!+ACMCYG%gMyBaVBUbvN~;^I12Y4vyg z6ycD$6Z@?@R&V{0eoZLqH!>J^}jASeUCI=x9bjAeJvD(Hs9i`XHxf?Sk%}}h zJWP^*%xNVl$fRW>PHl+Qjm^!y&>Zsd^DC96%=LLJ$!~*ADO)d&Y(Qu{L+dA}7s$C3 zep~1pO68;acLbYu4yb0fJKoWkk=Me{R`F*ork`E}`-5CQMMQX43_;}-y8IdbU z(juvn2+Ht|iHRw`G?x~_9^| zfcCZCH+0-WPqVUAjvuc&s*2*^)alc#pE&2feeg*ZF@HdmLENrmG{{~FdlIwlA{*cC zD}?~m={5uEB){V0b#C{*AtF#`2?r6xG(r_qQ+L53=E3NU^rlTpii(Q0%6oMv|H~J1 zQZJTa*|Mo7{n-xP*Hb8j0Ms~ccRsmEzP$UvF3Pr-UU5N}RAQrF`%>)T- z;EIgb=DuM{it~tze}79s5caPk)Np99MDArHcknEp)3g3zCI9&SZuUU5DRnT~R9H+t z84s^(tSJQmnEU0+CEeEEzrWuH))YacWY}JkM(**M#LK9;$S+}a+M5bU`upW^dLM5Q zGMu){3C33!^}g}%7`Esio+yQ{EtJ96Fa^i!7K z6?1je35+zE=3e8c%Gn#KMZkxBKZgpQJ}%aa{cXt{*UI?c9f={N%fh|gl@keIqEP$= zfijy}yX^Y2M@Ev(_m$0_TC;^ZMltL^(N!;Xrx}JiDEY-;Ah8mWb zzm}f0NJZ3K>E+qq=0myQ(*YT0&3%vyb^%tN$DaE5AdFsr zi3DS!pTw6AmN0H-QNHr$9V+rJ_PAZ?kDM9gI5UuP7sW7iTCC?!DXHzLV3zX;QOR+j z2~jN#boc!c5)mx~QA0>%-1+GkUsaZW+=p1s`}Ugq?xbSV4eXy+Zh}8Qo7;ecznE&? zi}qll(rLt6!?(sCJI3#bWu`2>XYNJt(|HA$LJyP^i(D;Rq_;wCM(cu1U_6m?zx+`g z3?ebZ?ldDTfm&L1Cl>vF{W^wp-=o`c2;9aRRaI3Lyz-h!=;}Y#+lsqlx3uqZ-qOa) z`D?+A3RP905yeOiJ^d)YxeF|8hnLq!GS;g-)~>~wgI$h_vYuX4LTed_KX^GxAyKDL z<^atdP&uU7&aFGvgs! z$>Na9e@u+jZEhn=-?(xHbzS`$*QU3or#5lhgCoy*&+F4^rwD=%-3_}T$&y4Zvk-F8 zBGgWjC#pCfBPuM#baDe86k^svHZzW(;s>(xGY8d^1f>VhIlMU%aHCPRVSB~6DH zfZg!rx>?L3?7w;Q=30o9$U@TG8e5Hks_E%z4jDUsP+6@V9hV`OC#51Rvx<>Y=EVHg z20P-zb3lbjwmlIf!eRzt!EZ##+=ZXn?-bqOMQEb!R5USB!xwkGxUzC7T34b8MdFHi z1(YoWzUNrAihRb%$SB)wMgh;!at;n(r0-*Gs`7`-3PXek zBgUB|Thg;QK!io1+_-VWP7?8@Tv*Ew19HN|kFG(lSAzZ=2H05mOWx(FIeTGjhrWS9 z`IEy22JOw;-c8&2sj>aBy3>^mQCQF&Z* zdK8y<|MJuCE>s2$9$GQ~%N`yQh_1w~T}UAcl0$=w@^a~(fbxV&xlQHnZ6wvcn&!q> z3>bV0A){0&K-L@5w}G+FhtN6f-X|!dfk4rH-}o1RmT7nq#Vk-pBE%f~^$UXl76Db* z0XVA`A_{b(f@9yN&!RgcT`u|-$DyueNLIs)hu?k(!Mts_)H;B$T3mtXU7z?wvU+ii zAbs%$SXDeh+q~Z-$j@&lM5LYmU$KH^_D%35kkbbj}MFM<_>jIEhR9f*UR{PF2Vwq4#!i*QuIa&rl#8}Zw~&~YGD@O^!~c&fb6p%yfBN{058uPfojkL=)GhFCjfa5D=Dk2C*S zR3pE{j>re;r8Cz&nR2V&j|=zWVaBfyPo1_s{Wg8Lv`1&rTt|+uh)j}*G!siRam(x% z-B!Dd&(D2Wvjts`eg^!V{nA5-4&J;{P&92lJrQ4(Hz4gAk)&6_$0r!(IIQ<7xbTw> znb;v0AHYHoA~!RFFNL-|xUm{b4J>1?hB{9=HhRmqDVNZ+$GJ|8e*n>e6M7MDvRB~4 zSUCp<7;JERYib-FEXOeHp}MWwD$2j_s~J|ojfr2_Yf|(xYrvj%^nn8%Eu}E34x7>sDVX^ESkw>AILla_&k=bk>@~@SQa9%cR4e&R;ytc zd}2TeKV;c(V>2Za@z4$svl;REZr-d+4zydh0&zUFLq8xx*D`8w+r*8SPJF?UdH7fM<*RQZD){A$lIGBB* zQ|kQb7Ju&KgI48owdkEMhnCXM-n0(uZv1cNEG6FN?_I=p=D4tDMMXA zdKy%KYhf{vwr(QX=dkcHVjjg(43)zLUq~TlIKZhp;A?>V*cYp=u)12jx?30M_Xfl4 zQu&~#0TCLcD=3jb5BpIB3%!@EdGDQp1|3NEr>fC0{WB6YR|sU8TskV%s5P6jas z$A~f@MiSn^#J8I`AZV+?VC-QNH@X5j!oH|2%aDt#gn|XuA__>l%GmAPM9Txn0mY!O z!_Lf*zlnoFWP#JO*`R?5n+-md^b0AFP%>>i{+dZ-Ll{wyl4P1g)7C}>9>KfDU%5^T z4S%BDariG}kD17OXTzeXy0+>5PW0?%AKR7npIfdxRncHFob7k==Q%kDT_ThOjg{{p z=nx5O8b-s5clbS58z}pSvmDvEw09eV+e3B~^(*(xsI`7x$@?TYVzSrj(1E_nrB8xZXFAD0K$6oBn`p1C`%~dUGsafD3|MzG@7Bt2l{mpoNwBuGk!Qg>DXi zP>)lOy`lDNub?`KF-M}c+|t<|E?(T=(jV~!lZ(aw%~85@*XYhuVjS=-T_5brMZ=`o zCbjV`8I!m%mC7q`kn4P1z`QE-`i!md{}TaJ|29=YM{6$VsFWe0 z=HG@(g)RP*gfCClVnVPDUie97_0HdkJ4AEtV_B~U1PY>Ya>1%@Pw(^p)0 zGFW_V&C!nV#G&AjP=33&sR3sDotV%$Jk)I=#FsQ@ED%>4iQ!dz}D z->vJBDdawTVsV!7hcSv$tc+hL&rVaH z`y>X_;b*z|w-3)~Sw(Vu5hft9{^Tkm>a^I{Wn6M2CgpJT3O?_pg#q2wkX`YR#1sNZ zjE_m;{u!+(4FGQ5LzKpVWOlbhCMD>`yHoF-8g~=^*Il-KZ*3uA@Z)+`4x&U|O-(9EeU;BEnw=6` zB$?zm!@_Uf@P1jcNW#99<86xW>hqnP60H*^8IMGK;7ktSx}~Vkl6HwSY;akleI(=M zZf-g454xxo#wKzi!|&jYuD^Ud2=YBtBvg>`WY8axNZ|0&I~j}XL@+@xa73yPy(NHz zIIwoaOnmSl8^)y7X4{$>KC@o}qL`$BVMJIBMjR1x5Yb5J26sWQ0=oR*2MZy#NPusy z--W-t8x4JeVFI^}wrLpp=dU$Amin)kN}8(r=n)eFSnJ+sY2H$0z|pM=i~3J{RLDb2}v z{}|m?ofHZ^{a(%wS&Q_d8+uo+kq&AV>3a4fiE!00%8UJiv%>GKze=| z{TZ3vo0gs*a^DONe0FwfRhlop51- z{u9eu8N2%4z$_Qo@@<=rv&}$TK1@|d+&Yj&_&7)FSQ50 zIZn$io}cvJnXqm?MN2zG@?zbYp0BG-PqjLqv=~hIRtBEZp`$^UpTD%D(K^QbG$OyV z&-86@ZOmIT6MdDmr@l<7Vu;t!XUm5YSV&b}&0F`bR)nm{(o?L_=E6@ybc^4dNCS0< z!OC;n%U91_Bt6$`gz0(uFV z5i?NaU-bZe?7O*X$;{*sW7DTknV2_BA)*MZ9}MUM9|T`P@WqrmY3Bx6*)#Xsz279O zmy=jFc$@t!-&sMQWN9`V7Wh|KnvB*?;fcC!$OG88ery-;p^g(9JOcy2Og`5WfX*vz zNY~|5bN=epZjC$_k%^;{mfmREVI!x%oZTJt{5{ynqomVq$B(hB+G5RnUX`w`-pcYg zsOK?svm}`13ioyl3uGg}Jto3kf#Qg4Lui4?C>xN=Ui8P$RqSKf91{ApyA<7lM)sS@#tK;`J-K3ed_W`wk1|M^JeCg zC(1fHEbI0~xTBfo#a;xF7c^>N_Xc9NyXBv{`2?FefU(QKM=DOQhqnm=3^K{o1idkY zO3`u->Q9@{=An%&@m#WkjMgR$F`64PG1M4RahMMvt(gyeQyE?gsJA7MqO)|(WM9*5 zARS0F)~5;+JB=I(2h{>``4wVaWN*sJ+@1|EX^}qLSmE0u3!EU z@+X$0zDxBxhg_&Z+q_2`Tp>;HV34^v+|fC%yKtXKt^auMYwl6^-;2fSzD;|NnVl z0!)~ABeyudIP`|V`2|X$>~}-m>_Ty*Fd-4PyyW$I8&aLyWo{-l`}&QCg$Xg1X+2$b z$?49VIr@Keh}myS^$8VehwYf<2@z*z359~#5QLVkm6X){c3}3#`2Hx3*Iz<-jDdmW za!bS7K!A~rdVNn@)f8(=74f8EHO}St88z7i7`c;&!U!!_H5&$-7{Vso}WpGfFD`V-~mxTe#1<8U(e*zAyTeu%&B1_hp|k-Lg-|ROSJZHUrIb3o1g39QvOY6R!3g*+>hT;VYgHq>Dee&=RGc09Lwuon={-0O& zuWavV@dg_M0k@mLP$cBvU>H4S3B_Y`f>DLoipYi!4E5s=qD+NhmpJF}86@UoI47(T zi-ATFApz3CFyz-d*J%$JPy|R2%L{QA0eV9%w2(psa)9+CwoMe+93(-((D=1NN==!T zlYMU2sRrVUpkp9)s9*(#nz|uS0A@0uG!Sqd!ZfVBZ{L+uPAI5AU|~G!CD^q{1}bo` zGO)DfE~qxhY4;yu$R0XFx7{V{asgaDMvh$ub4JwP%*?#(=^Zl-x&0YQRRzmh9X?#- z^SI2jYi9=Ulf?Hl20QQ5ji(u$V9jDp^GFc(e}9C%!IDc*|Gt5ql@*mntchN_V(0yo z#bVy~K2Md@zvVcv%j^2JiL*xx1*&B{qmqLMwso&$V2Idk#YT`GFjC|?f^l@VY7x%; zDQQ*P>Go+kOB3tQ=om>SOMcAj%{V);B`!f|^GHIHmO$&P2 zf4sw}>j<(tm3vOAT}^-DZ&Dx>=rVF#Az!q0sx~>}{t@=`PreU!x-(&t%#JTl_EOxp za6K#hRIz%qWkb=cM1>2(-0+l7IG)+A-TJxQ2jAgUb#+OoUt7ec)3R6g@AC}QHt}c? z3dH0jR5mz?2(t^HDH(%K6wdHfUq-lk0_d{3p^}mkTcmtU!twec*FhY*d0*gWLc`kv z0s@$pFDLJpQ^sxq!Z!#$3#E*|zyAR(Ee})?Ad8t%7?i?1MJAa0D)Ew`cVte5kIy1J zk3{2$o~q#IMb|+pI|&e(9{(zi2~=!&LBX(P z=U<)Sf7((bC6cjXk0!$eP2uVUS|k<$7$NV?VPzUc=H6p0nuZD`baFgK^!?*;Z1|xw6v+eY^6|ITt>Mtt$oAUvQ1dQ zNKD-cz!x}k1`3Z!3@fZNm4p}@J@CPvizw<=nr3pKO1{C+k5wv)e(nqyC1m!sV)>2X z%Qe%Dwo!Iq3h2t(5R&`;*rsI- z{*@va8*uUSff^8QvCD~Mz+vklWDg|+{%L!}7vkaxZrqCv%X|$g%M$2dP?(bXEAczh z&j|BQS&R02&6+i;$ge{Ys|ijEVRU<{xxn_L+aeDGsc~`MU_2cdo}$$gW^Kh@=ouw}}d*^0i^f$a1J$s_<@{8s(=WSVK1MAxV+`{KkSE4K; zSQ@yg%E~ow>Qz3x{graMpMTq$(TVgiCtVi0SoXpPf!o$qPth1OG|XFfP{H}W?!Km@ zZtWOD<2%)<7nd-30xK1I8NcBnByZstNJG_e5_CjCUKt_hG<9@5gA~S9$2X%1=w#d1 zxkpV+5qu$P@x+!gz&7w~Kv6FXGG*YM-cg>lYbgjsDg_5|E%MNy$vVg`U%r0r1wT+_ z^?A@Bk!_VEYgbK8Z8lyT*WS+S2$1eXh#^6%D{f#5zqlTP3s%et6+n-Vns*-5xRf2c zcY9Yl85zwO?SK5_2`z;@4iu>H{9zs~gsKs*j_eSIhS5!D^IqerBM$t?rx4fj;F~3; zS6Vr34p8M+g#9t`P`T8_ky9l3)VVTNF1KmO&$oQz#V&jJ2TZH-ycAVW>#_&rN4dwx7fZ|Y)i!?04fmrx zd!i#(W+<%d_**xgu^3Z4kGv-h{G-PIM`O2du5b)nVf$zdg3yLApBD6zFVz%zn+L3SZv3wIfGt1o?6jTlPtaGH z$6hgSFv{bAO`XWkrCXe}Edt^#3eIOWX%-_ZL;r#>p`h~Z8TBwZ4f$LheXLj(IiLmCQcc1#g z`uX#*1cy;Kl_c=*?UNrjet2lX7Z|h01Fyq<2`k@NOuL@b#fu%F;fZDtQGJypfs`K7g>ze+Ck zAu5DHWQX}tk8qlKkRA&nzb=}LQiyHpU28TiUjmT(utBZE(p zvKhDkpeCIzFe$n5y^hf#$b0d(0hukQK0eT*mArjSc@T=gYuB$|Mof(9&Bwy%Zi?q-eE+){zE&t<- z%@t>Uw#&$8JFu^meq$uBE*sEwf4=SZA%^pl580%(9<;SxzNnfX;_0>V#V!>U#;Z|V zE>2ueTZyG)cJk!n6YP4Cj4)qDMBcpn`5C7NW2|c!kQGv;fwd-D2QM!#3}}TY`S#;k z(84}q7PJWD63G&+CCMhZ=fofh#R;ifs%~u)*=B8Q?EYKVgaki&0&RnZjU$>kNlC>1 z(m=i*NRZAGCrU6Ck4U<4At`zKg;{GW0>G|0Tfl`FRsLcuC`pWB!tCLr&uF!w zz%e#6y9~zF6e22!FWq0wh@0mlZg!L9+r_A)%E0PVD9{v=pN!KsC_0+c87;!tU>SGC zR?p0<&#$hdCS8W$deREuN9>09YF{jj^l?WET+cW- zsNkjJc@!FzN6Gl%p*cM$Knb%KOFGf3l@J>YFdKHD<5Mo6;; z*{vxqtXVt~C>F8lqEOyV=PS`Zhq7IJ<d|b zYX5UT{&bCe-JN(t#f!L|-4Sa?&y`q5>RD1EA!ficFn;(}_~ICM-*P^-w8_JPYl%i) zWo^72e~ZgH`vqa2kDu6p-?R-4re)=H6wE$lF$Nw?ZF@6^tSIUFDGy%6C0-bL|Ax^N zX}@LXxK*aG1cRlqYz?THKeNSN*n{*x#NX*gt3J0K+Z6nxeW2IBZNP%JliK!%wWJMV zY<$2eyzPzJlbRc;Qd_ooXG+eP%$eBlu44Ejt_BkDe?P1tfbnR|al4j3*Px{;u0OD? zzE6Qw)MTzNHv$4HMj^u%CNUT?Sd=;D^%Szz=j*zt2lO{TD?awY}Mq-Wp`-IUvF zxgS_tM{Ko7tNE1cUO3JtO z6G<8%K4?|Q&W(;J+$j(zL>ykCgoFh8=QB@!QKu}1Zj8y=Z<=g$UT4pe__j2Qc9rz$ zg^dR1Ob$1$`82`kb1R>w?!l28w-OSP7j2{$>Gio37wqmARzDu`q~}M0^dff$FY{wH$?r~Vij0aq7s*QG)gLH(fJC4_F$#-p7 z*K3Q49{$zwhD}tnzL{o(A&|Tp;zjQ@4XLSRc@dxOUs_ta1GFy6H$v+ojhE1{(K2Tj z5{LjIYtp#){(?}j6dxpQJHQrJ2o6c;9c8V!I3w;Jv63NgXON5-g6fRiTl4%AsGjTQi3sHfKR#yfjv-W`XE`S z6-5o{UV+99PeLQK7S3eA6s2g1h#r^>K}C40RJ9z=ShTkIIrOAy1uN?p5&|nZ+HsPs zf?TLo#IYM2%Z=dn+|ltF_t4nv$1G*cBk==HG=@#v62JQmUXXMb7omqorQe zvGfsIGOX;Z&^3sReHdL~K_b=^rpHzo?voWamg;=rs?9lZ4*-OY?wA1|4@yVgzVz*y z2d*)u@8(|2rD6hn;Q%`sG$d`ahSHkbpT7&5CP}>P-a)`Lc+ZPaS`o)8`pb+{AEn5| zCJMxw+nmRHlPT_3S4+18qah=A(Ew14q2v0~cZuE--LL_|I@)E+mZ1tTffV^=b#+-* ziz|I=OYsDJa`>OmroDh}X2<&?aJiUV6S!e+V|7s6G07aGmeemy<5grD7tI5yM>}hl z1*p-U9P1j=<@PHN8irZ)nuM;Sh$MriZfrQ@0huZZ>7lfK`Qne(!le4vw*G{PJ+I80 z_U$T7p%D}l_8^?|zL=nf5p|0DaaWp*hnZeEiJPUY@3e<+uDIsZy#=2YKV#YXqi2wh zl-zZ7T1lS(ZDJDA@;@!uLCJ5@slv|pHFeeY?|Gs3yg*4)lV^C@(hTE4sjctMvdP{k z)@RWR2K1@AT?1{rqT;+OS3di=CnYC$07WC91X33(oF8Dx$HB}~=fOcivcSSj{VQm} zEHuBhwb3lQ>b7V;*>KPv)s;a;O(u|`bUJpc1xaXK$S8R8MmyCCh{;hgW?X$?`#u%>g)R!7b`|?$@C`knOerq z-7)-T*=SbvW7*5<3hFDqMH$Z1o0ObqR#-O5#hvpJs~h?vD_c@oOt-O&v1}^W!Prdi zaN%2WAbojVRLfLw>eHpB6V+#ok8<*^(_wUlE{%9o3l; z^5>ovzG5knq0S>_4jA-WTU#mE<|7$_F(9IU2S!UHYd~=5>uPk6fOBvM_B5ihPb|5U z7J|J8*xVVEyE(Dc+SYYxVWlii$3SSHBoNIt)@ic=*h~Y64Q*37QjW9$o;r*Mc4;8e6Ql zTJ&qAriIm$*ltjI^KtbVhC$Vi@D;i_QEPlg#&Uw;qPd1fK3CM7d(|p`z&~vyXA444 zCU*8w^&!7==Q@4V*je;?%2GD{`sD;I%{7JT?pQfEUssVNFDt9{crybPH_gXdOEipi z6V}nu{RrkLSX3`|i|To3ukB197ypa?qDA|oiawOGWZ5K{PWdaR^nc_SQ5REG(UT?3 zUZ_o3Nl60FstXs|x_?Rv3JUHxa3(yD} z$;s)%qY1|7Z7y`%@9B%$Z}Q3%WOM(!hR&X00JH0$mGJ`eLI5KyMRKZ8Nc)4J783BO zFgdV`%sa&vK<1sU-ChlR+XVd`3GtVg=LCa8?90H3hN`Lu(PNq6&XSV~tq}?JBv{qP z8^gRLBu|sLtXfR!oIyECDbm#U7S^<1UX5q7&HrxF^b9~90 z;ueC(bXJTS+< zTYPH#+@hwYFJ>REEeBvi{|6)b4H)2%_+3PS#nIh&u2-6kI=maG4MDQ5uG$QlzmLZD zDKcsL1&a1ol-y(#qz{l4y`wGyQ?fYCfqD?dB{7PCSvq!W6dN!A%(a;Of;hr?h;Rft zwkJpfTQ0Zr0L{U4qh;VrdU|_xsX>Q@-k04|4xc%<>%bYkw`b44_edX)?6pe8U&!W&|u z4pg?fBO41eK#ydf|4FAXGNsgz1U-7t_UQ%e*Q@9w_T|iiZ+~-Pzu>~^JjlzFCsuuu zqylF~YhPa=TCtrtYB8oGBO`-Cfz5h8E~F>&JBF>E=jBP9`=W#G<2HN*P;U}55oAn| zciz=bCX7chdL3yZso z)n9S*VW+|LAu<3@irasB{>`nnyco{GJH({m8n0Nb8Hw23n!-=>*K2TwKlP)Uf)U^} zy1MFregB(T0sA$kZh?x4ZCApbJKQi1|*1 z_4VukK1m+~YohIF-;qh^=@v6EU}9~x+=;jE!nlQ}@|4=RpJ@H;t>@Ox=VD|G@;=$q z>lIsHb{jWjXVH=Wz2zixwrj@*wgl4uk9=E+gfy5>7ooUC*GQzq2p1uv8)~y`j-q|( zyI+mL02voYt%(W|y{Za?#Fp>x`5{LdV=$9`xTOfQeR!~KP3PyvzSb*8PQxm7h}gZ_u5*_MBmbAZq?m z(#*EZWVa`m`dwW0R_B{@s;Ix!tEKXSGD20BS}wL@?E;WOzsaBf`aRPaTwZuRo%?30 zrw9KJXNSHkL~ahIfm!a0-pV8+BSUeAhNBHi2d1qnb=PBO!$k?T5*HR$gub5wpuLIzR{*BF^&PXK<|S#XgPkC*ew@4J4c9|La9EeMSlEzZdiV4*=b^VJ>J}4B#{;lmg5d<9 z%-CdaX|*dy__tB>9n7q)#fC1<>>tbB-;72*gTyEvEg*7T#j;=3k3AW7=*0X)$TOe_ zI5C1n22xHDiy`u?^2fi3#3iI$0eaU#zzeH%D7L}8tlz==jszH(bX7*eBbf(0)Cdp^ z53?W8znW*~tWH`?O?2;qP=WAtz%^)A2xlSe009^CNP^$-s%Rn6F5to$j)SIb)K0`$ zn&;azV=@-V+Ijq1t~R+J%Hqe4Vs6e2*ISSX6XOWc_5J)wOCer+v>^yE^WU)~tP%7X zJjM$_UK4p2W|4MfrykKiuDMUgj|`zkqjxYWPC$1jW>bUJUm@$QR*Uozyra$rZ7r?O zeLb}Wn7QRCo2H}1PVwZRtoF#FVkMy?$L8d~_qZ4kadM?!FRfV=+O3G62jt{YbM@7^gJ8Qr4zg{~Ga*5MQ%(cK552T>Q) zuR%BjRM(5Iss8%ME5QN9M9u+tXb{)5m;fCem@qd_K-=iI?1gaUNKeR7Xv}xX-SH~4 z9z@?j!ORpRnDHGT!H_rUhnH4_Qw>-mi|nmByYra+jzQ>%ka4_lq5S!KFl@AHNqK-n zF^j_s$n<;)#7$m!!NmyX146{MHRgvMysKE&nD;3DCLht&h)msaTc3jPPfY1D51Ak~s-T_A-TVh5WT!AKWMK}b=mKfI|Bx>ZTIeU|cTa$s)7uTXaAxL2!1%(JR!Hf|=wm>~O+NkB1 zDMX8?Dhp9g=-PGDKTz1O;|Jdm-CXosYNgjx+zFurQk~}5YOwL(wK6EGXs&E}>Y3`F zlOu!Ki!>NzqNJ>B0{cGVWI~%GpV@YJ*D!^>$E%jWpshQ}$r$1K?HjSv0Mop5T__!V zL4~2GE#}waT?OB~=?_-^2k*X>kT+IrTgU7-W;48u6ydNSky4UEl$D6CPzyK~qWkH5 zzA;&m1R|fFbix+7giF~R3ziHAg;j@L*1x>b4yEqoF;M53`i*tqO5Q0>hgv^ik?f8 zXO-fjwW(Doc?RH{qUYL`NH8K?e`QzK{2EQ2g;}C^vmpV)^Qnm144ef9?-gTNdIcsv zh7>)+luAd$g?J#YKcG0HMytxbI3LZAJR%zl0|FAo>DwPB)z=jGr#~v!%&LU6tF2qT}^Am z#lhqfHXm8)+ji?6x~`b z;m~IYA2FHMfw45P_PIGE!UfL&E7p}PdQp&BpBrMKdczOJ`rn+FxA+ab>E9Z&7y-yo z3Iil;R$vGk!EqpYnZ=@E=D-X9mDXeBU!hl-J+NBxMICr9SrvxffA-zEh^#9N!sDVy z7#C73qEz|Y6%X4S_%!l7ZC}&V&jp8Ii$@}**v9T>xa=T@3$d2;S;C1+4Qv$8Ed>`k zl%f_BXb^xQtp&1XkLab|ZpiMaaPr;>g5ZKg*oPgCdm=o@d!^kmGu_mifr}2iFDaM- z^>^yrbU1*gtTh_^qCM`F^cpvpiwx)VpaKOjZ&K(^Q}=Fn?{^YlK`imTy?6i<@mm{4 z$(%WV{)UAHrtd)$RsPy1RaihkU@NW_Hj>&$Cp6G?cHqz>rkfmzUGBiPhJ(I7oNlVC zuP?)SU@jINat}LDPJjeSOjzlyK7$Pmrt3RR^8!GkL~J>|ipT@79*>_q>5HR3XLSCB zTpUd8NY4~57@{Sds1~|0Jaizzrdt(4>EI=PLCCn)*0uI5sa+&L4|<7qY<6K5Gks7I zL-1wz%zn>p+!SmBWZui}-3#gH=|Q&5TfcriiJ(Ncub*k^g;iZ%5f_1pl3UMx39Xcu zLt8}JQJ~cG*6FzBEl%Jt`j*ClCmQTSpmyL}4j+vby3E-B^MC2MLz;tN0qE*!(wnae z)BE&>i=tmZ6iykM&BT(Wn1oF9EWMmUAvgsDA=O+fB(&$a`Uxmq$!IOy%wfAt*Yr2A zZS}LfFvVMWBGzRg^t#Bc31>GcC*$=Har*Mex5hlDjgZlxJe!0Ol!U{y!y#%oe>V+@ z#zlh+HgbMW%UN{P^Ab}zP!WfnL;_iaoP!vU2hm51nZwed8I6!InJ0GSnI}3G_9dD4OOuW{P*B1Fr#7XHp_`7fXbM z-vMwAws1IerVv?_dLk0{A{z&|0Bd>#4&~RxBWeWI{QgOhQWt@ zeSK0BS}06dh*}*jm{(v*s8I|feO~sNu-I5e+(TfZYXN5933!bI5tg3IQBhHqU%vz7 zfGDN=y-O#Fgd}eb?~=yRHq++Yhs?YWqYH6OZGq{c>oj_D=v{Qy17BfO!z=Y>beH?A z%-@bWKoX8t zp>)6(ubyBm?e9jIeQmHMx_@({#m}MFf)JEe_0B31PKt{E5Xv1d<@9!JMgykwQ95gwqPc$Da!Rn)FCIX^z3Ohhe+nvo35M1C`3;9&vu zg0$%DW@FkIP^{QAUf@(geH8l4p`V1s61^5I`q_?yyeOkiKphl%EU=vUw< zjbv3_;_L5Uj2(kw3#NJ8 zi4CBdk_!TTy-#>}cvJf^q$=P0;yAOUpOmBkv>LbV79|Mc5^J8neVPafJ??{G}Db zi-7SF8XGE0WQ=SWl@C_+1Z}NjBJ~dLQ@$a66xtA{j`bu2eeRNC)_gaCaRBCP_VP0m z#1335nFU5P_BiAUK=(xkNI*7~w|M9#s+2hF)irf>q`GQ$v_@W7nqZivt%E}udMhXp z_&2&uVZSDM?l|uR(KDeJ8m;P1ZSXoVyCJ;1#`6KA$xt<~J3{*dM;Us5a zVOh)-EWDCz@pxLifaJyF*pvE>7Pja8{nb!HcrHmgW)#!iiywl*r36059)00XGzw%| zxBU}MRNO@6CaV+i8HQeZkX?#s=#VISY}%K_Psxxed#QN0Nx^h-N!sCX!f=1cIlEp( zsK)KZ{d+8Do80o#m`V*bYJoZGjlBnFNnd0@aC~Lw=q>+U!A+0)+_}Y9$zd z@Blz@$eIml-^*98ieNycRZp3nI|rdHncaY-?BeyY&hxmqVx+3C1_h>LWtTY(pN;~Fo)g9m`H9ks$4J3D5Sx+gzm@Em+#D3rBK1w95f+bdu) zjRC-sKxhQUE>OE0jp06*i5XzF3Ttoou*v_m5GM)C)<{dIb0TR#!?8PF4X0l#inw@K z8$xnw^dNKxI;4m=dB{!_m31z_{y~Z@3-I|1)RPQJ(FNN)>Mpoq%MIOy zL`A7+T1mc5*kPq>X($2WvDfnAeojyHii0Ga?tTUrP7JT_deDOJ!)oaw4u$ws9^`i}V7z3iwOdU+jza2tkpZN^pw z5|Ia|{t`I*D1|8XB@tAFWk#Ya=dIV_B3ciKXtEw&c#Xo8CXd7KOmCe)Bz*;}b4V~{ zLeQP3e=;+f7>y2q;I7zon@k^YitBA<_fsO%4dld_{m{$^?RYrj?EvuRkgKZ#QPR5% zStCSRugZBo!VPgiUPjz3F$Ewrk55ogHOU2}F!|Z&nRxa|2xP&wLZaPpBKx3PY&uT{ zRWJ)i{uoETtqIyKbP~OvNc!LK_eOTC6NJ!8ke&eS_8GTtdJvYAp%kz!B<`DlOxk0% ziSqCud9FCB^Ui4)oqG%IEM@?BPB3XF%GmcThZha~GGFaqwNS#fYF zww2LJBu%13az|sAq!AkkgK+cDJEMK5E$gaL5qT3P<@3wnF=X`8P}ZwQ7h?XFJMiG! zXw?;|I+p3_izkKqx(_|YflJbgfF~*8O;IRrGgH#=Rt%61``tTMA{G5@eY*5Um=WQJ z1d7A0k@^m~&B=zxLWJbPH7dJ*FC`V66hV8i&6)S@F8uZN2U7W$;kIKS^IrB}!)O9v z;M1{FVZy%MABjX;GKUJ>i5)x7?H|Z6GKJd%^Cx#i+U$$go>X*7qZ9;Ext`m=y@1!P z3=JLq^)#To>lD=pvj71V;gt z7HNipkX zhg&`U5h*)->lfy|psb5^9Ty@J12(pWe0&Mm5&&FcOXvrS^8%?FSQ?8!ew06#L|UFI zPKi!bfc?%7o2mRdh_fg}AkOYu5+k}tIcr`x&`+%nA1t5dli9byvQ*q8N<=ncQ zWK^MS$Bss}z5YXpnjztx*d_^5g2JNiC|f!#mQAA%2nq>>lrmZ=uRz6j2cVC~KD87{ zo`4OW4}!s|5mrl-E&sYPSaktDioAmaY*$hN`FBMyCd@%rgP+-U!P4i_h$Q8ZI<|QC z6%L+HHuXR!5tDc6{{8!EV(vPuL`SgU$TOV_wf@cOI6l#`lL3q74e)7^?hbuEeds+? zf|T)+BPQ1&s3e)y_{0exz#}(u;(4aNqNyjG8`A4UrThEN8A^OPieOt}Aw=u|((Uy7 zoB+i>yD2qQfgcWS2n1c>eLT=|@u3myywNGN!YPs_pi0lit{a4FklH6!%Y4?a{=jDGHPTQKS z{~ps8Dkh_wU!(HLfOnO|ULrMdKF-q&o?YI?=!QR zD)pV&&$BYunS zfZVhPi02m)2!)$F2w@4tQ=nL;ZGc+Injy5Z4E=3eY2S7WUag5%JpB8vDGO ztpO_QKR(HX?Tdy&C|go=W8VT|mW)!amwS7S3}QH*XG`uF|7^ z-xFE{1U18<-^gO;(!A^y6rRn6bY#{KX--@>(AbnGA#8w~Lwh;IPyU(@z$Q~Ps%0uS>H93K& zGfymeb8xh#Vs3D`}ofHM~*Ccm})|qq4?^cfdd6uXkAZhBb*I; z%4U5p(G!srI~CMj%_{2XW@*c`RA}GQ$B!*|*3Iz4+(?SPuSXus%3`xqaJ zUW*_Xj#uivf&y;z){u5?o?Hwb(Q<0w@u$-%Dy%CYK;&q+HKztVA)PAD8Rogrd&10R zKl`y?#D@c$n3=J`p0|z&IZZ$bisVAj7bhr?vq6XP+G5Yk9ZS-E^wGVXZ^fvjQ9xD6Q8 zXQ;T*FCc-ypK_h;!C^tB9!tf>l!JDd9mt>!{Jrvz!`i?2dG_vUZ(f!I$A$;oZ9B-! zENOjw+LoqkcgnNmkw{|o_4Sc^kI+r{I%5a4g{Wr*5<_0~pjx?k>sG4)v*Yp?ycu3s z+MJ#>CHn>*BFMlwaAoMv;DdJ;%A`3YAtFLU)d!k-Bw@uvmtbop+?Ukpf(>I9-}M7T=XTjhn(F1&{^s=g^JDAlmot0C+x6>qvunhMvl2YcjIAuwX!2A5S z&!fq)92;>0K>-yGzj3t3X@=>brV)t?>wQi;#Tr0lFN8V`p5C)e-Cq^ZqYcRC6=n(M2^W{DTBASx68)Lr%!Vsv}neI z@tu$Nz**KouD$JvSN1XL2G})&Qm&lsgA?`8=`qjfym$~GzSg6-PAMD$akW1HgxXhp zw_YTR;7I8JPkPidhsuyS-j`B8c%E#UuW2o*z2#YZ)5vS=fo+tJ>M4VQlERDJT)Lyz zQMu*=sl6HT0B_D$F&oB8qM$N0$@@;y7)Cl{ZB{eWc| zLZB<$8S`UT-HZ{ZLk;8Hv#IgnGFyHJTsy??un8xm{E8Q;+GCR=L40Py(ya$r$`a*1 zlT{6Gtmx+s*?pTv)M+G_v{B9;>9cmL@e2x&Qk#d$GuY{}P;kb;fgb>E&vHG9XaE_N zv=h)5*`s4Ug;=;4CcALDp=f4>258) z3k(S>>2?)jz(n6C_N7k!<0nsCy`F!xl(TS_BD#8%j~`{j?ICf4b=aDhX5Anx!H;rZ3VkPy+Ah@ik|V&yTMOStq!5}@w2YcX%0 zP+gH05yZluBaARWz3Ow>5bS?R?qxQc@bFvjK0MV&YA~yZ#d}iz0vizbO=Tcio4c!` ztA~}7Ub{AK`0Ue99}+&rqy_^3n$NW->^lX`9!cG)aD$2RRo+knO+rv`u%tZPlIQaC zBku;B5_@*>1E4UV+covG&JbPj4sAfpNh!KlFye%zQc?YpnibI3##rr za9tQdI^n}2TrBzm+sHs(AHSu4$kR#n;0NOQ=K-Hh`%I&qIjw&^Z7uO-2GoqMzP=MZ zF)zs6L_f{}K98<55Gc%trk0jYWxnA#N+B3?u=VFXIPD8t>#6bH%~>5CttVzHl*V|< z`P@=c$HYzJ8fq1p^`09u+P{tI@UPD_ydB3C$rLOd?1q}_w917@!MEDl;dRif%B`bn zqwmYm2D&DaK*6+7ovYZrDwb?q8s?~`7k@d{V*jefhJazGxudLUp8*r9JX#B7K831$ zutA&8A2tMl9vCWiKDn-uQd(OsW1+DyWl<=IMBzMaZAtF}wx^Uwez>^#L^4?e_UzJEW@utUxgukh==F&5g+~X?yW_EEU`7&oW-*=O ztNJW{xPG!}nr-0yw4;(oopkxZ@?7d#@yqoZKQ?0duB+ugBpoR)FBiT^+N->6be>PYSx5Yu z&iS%EIdsx1wONQlseBtC;0tO*(f3KuC*|cME!vB_qjz>tYHnpz570N(*Mc3v`@-u2 z@XlhL&Xw&{U~V+q-~a6uQ;$sX?1o>teQd5V)YI31D$;VJ z*D=|LI2L5UF&E>3Cjfpx}yDFeUr@N^~;o# z8t)Mt4uL<>X}R9HMb%3t{1U}^&g=yb(_EX@4w@&IFI-w#87)#Bh*KvT_8@)MQJ6F6 zIhSW0@4#od(Fs#vDsk|8 zd}r>Bw7i&0{^95LMMrs*5VDJ~qEg6UOBwedY>?CndFRMPN<{l44!fK_v5{0vg;ud! zR+Cn<6C@hEar)bdP8#;mKrC!W4+82j(mpb^BAs z#fxN?YFPQNzs3;!q>rvRlPiE16m)mspgbgcQ8zRljhyBN!-hr)sKZyh-9s(k`^-uQ zamVBg+Dyr8h0Fg1;!k9(%gys_wmq1bQ?Qj#1)PjvT18-B|BiipqPRfxy2fRQOH&zE zB$(`{AKf;TfG)AB?eIDUN1?r4QqT!nT_KtQQc?UHEm1Cu%ALcyWXTfUotv!Dl`EBl zkgkWr5-vD%ukk19ykB>7=nP|!_;XAhA1=+Q{2r?iRmNVy4`fGYnMp>p7c6|fsHh#v zvpT5WnRhm_z`#}lfnLJ+H{}%-GeqpeWIP2PyE^YHc-j-3Jy!?Q)NrH4!PSMXjHCzq zKfc37B@^B<+N%7e&`8n~^gEgGZKfbH4$|DwI-?B${*uqkJ1CJDVh0H9N*zX8>6|0u^?8& zab8Qo@Fk{rjYb%}@@=J6Wj08rgZ19xHhd9@94DPDEy?@lcvA0-|=X6p}rRpXgBeEY@H8#vQ zF#+j%^Dpv+9?9td;4a5I%x0O97=>{dWIzBx*%=FQ%0ZE_qy2xo`CVR=wUd{ZFO2jax77 zGc4Jp?`M4i`qzcgZ)YFxDu|t_m7>2Srxfc6#1VFje%n@bE$y3V!rQcINj{-`sPGUQ zSMknBYCUES%$No5d1yvKo8&OsfVu#E}Mkh6rw1Ltw+2#^p!XS#+NIUlsDaE3UR3jo*b<<_(D(?q> zs!x-Dv9Yw&reH9VhfSgl4i-vy=&s|^88w!|380=Nym)G_3F{&vrh-hG?MunX#gZbe z^N@wVd?1uM<>Ea=p=~r7F#hW~A|-PhoAFPVxLUF0NFN>fd9e)>Qzbb3tf2Dsyr|A| zq`4y}2O+JfXc_4^PG-kO0Dw}^a5)vI{l7qViVRj$)aEKHLx3T;RVKqT#S|UkchJG3 zsDyApX_()Xi9>Ez?z}s}5x&%r!B;HKMVlpy=`>~V^NXCxtrzT;tJ>@2qIoy= zd$kqdMI#b+-E*^p`Eo+2>2I2_R0iiF3-DR^jrEV09A8GEVaLu?%wE z`w8El)S&=q)(5%{je}M2y1G>E*@lyzn%$xf7h^tg1tX@DxaQ)ctG*l1E-vT|Br)qh zi3d^i-8)1!E`^a!!fe=k_jP**?G+fd~0Z%1zK2gFWvwa4E(aVh0rn- zWcqx5N;fn#RMrt10IXnSuk^H)SZM9SGs4*G$ZKC@q84F)gv5tEsSpK{QAA}-RoB0QC#!Q znm1>N*8q(Vm}9nIil;0m0B-UoQgY&oG_tn(nB) z%?1xPCP`tYG;%I&2)DhL7yVW6LUM`@H|-Ik5HQBn{PjlS4k%cUouyFV(qO<^t-kKI zFqsSzP*r$rpIV)et!tiB#K=ENJ37HnE^B@%HEBQ{Ur&n9c#_-iQ^E%sw2W4qqzN2rD=;R1-+qtC2-M>^eFQHNP(;JddAzakGRjqvc5 z`X4=2;!h+ajSyJQ@%L9EcNf6Q77~G(2=6FhXVI93-#QLc2}4dZKyG!shS}y7h-yG; z_Z0yzUk-~uhI(J5vB;jAZmT>b>6 z(jIM>1pp?)8{bu}l4_llrXYI6$eS1g6guv6cuZfssS`t++9*f`H?@;&E`5)e*Uno- zMPW=f96ClBw_w7HE@I$FNZowWU^z67I8+JhMdq>&0NT_Sr~V~bt!@FdR$)uV)9Il? z6jqhj3#rxKzMJO&$Prru=VPMuc2T)*nDIOFekUzoek$a^TqmdS+%G+@^CdMp#ALf( zy~Zu8y~oT&$T6q$DgKXn?SIKIg)Yw{NoR*}^P2tRpWbF*%2|N(wZOK-&=OJ*0I)Nc zQ$|0dxR=uOs}2WE=#}Q*+lLZL6su{zCo^-?J;o7Y`lGD1@jj%WLC;K4N{v958aIw+ z*mb5)<~}m&XX)9JTq7Jk9KiRBj<@vNw7CF+X))gU7y8krQ26L*uM<~XirW$y5p{mm|$ zwUsMG9eCyUGk>ZBX?i}WS%ycjcI$-P6>;r+%WK%1uZ~Zrx^B)@q1Say#$n#6_v~%rR=j?*4w*%vzGv#d!dKv7PxbLL3lIV#k;aT{s zY|9RwL%0^>n;fOm;u8}Rrh)|_v64e}>_kmM7pLFTIqJ3Syo>U99vh@CN0>%Lrc`R- z_M1$Y0}Cg9N$=8GFjx^?bc$TQtjG+rD?e8KvVODxH(^bsnv0-Zk~O#;FTRcn;H&or$lWbR{wTteHjfFME-b5PkV8Wrm|i8{Ty|11l<`5zOGi^yoH` zG|Q+Kc~CC3lmouB9cE>DhpALGA5#bsEe&3VKgM{~?55&P%@pMeFK7F^ThL#!5~z^$ z6JlN72;O%Os8d$d zf*sa&b-wut43XH;a)4XbKD)ODDV)$`EU_pqd0wY67VfUn1@BRU=2_;#3fn+IN@dX< z+D!1b3~fPb%0(K-Kr!+RDKx^z(7q9!5P#wBM8@XK+(Pvf97k}FVT_{~#8$-6a2Ex_ zfZ{jj1&PBmDHObEhOE=25S7pw?Wdt53XdZYnlUO9{KABaXx&>G-LyTGe$s?{n2t^bc#ueKHC2S@lLE087Nrmu%4P&Ps7Qqou9_*NYlkKeD7?|7As)M=cVPJN;V!PO;RqRzIsh!t7q;*BVO&s| z+#bqg*M!}>HzG8HE8uK*Qixdy0Zi%ubVtZi;ym>TE&WFTnhADm6@msT6~BmXbq%Wk&8mzrpdZnk4VR~nea8Q zp01e7S~K|0W`#}rXzF=FPbI_boYgjW50avjO63v#zm;4?b4k5I+?oU^29i3GAv&UK zpwcJnfXbXH$3hMhgi-Ul^lWN@b+Z&j0+~GYgW309UvZ$NOGRt5+rr6Y4`LmPbS@v5 zCe~jVo3><=&mK9xMo02hc^u(r;iqsqbD8xbg@R5l^f}2dQ9$WF*Jn-)mswhS2-S-v zW-EJIS*;6Qe?8Pl(|*7=`!3l+l@Tex5$eMY^m9!;wgZQ=`Fo)|F_T=dt%SaNCp@1< zX_^ZzNi`bgbGQfzS$J}y)$xWs2j7QPRw7N}C9I>82FzcBP)@2YUI%-dqxE zYP$kWlTzCCw0!@IE&hAuXwx?c^UlJKN5+m+AJuSFti=ZCI%!%SFgv!$E$y4X*OvdM zuY}N7MkS$XkOgA96q@*9TH|;z?S!7be*9AQ3&c%p8el+Jj@u@wti#?fYTWMKe%S@k z9|-pM_iJrvn1e`&Z_1za-tXNgGjzx#tFKf9-5!4Jj6^y9csA!v7r>3;vCs;rq;N#f z0pEcKMQ;a$ae78)&j^q|!0XSwT3C1%0<*=Rm%Zk}-k<%< zfCsS%I%V$d?aX;ykgOD?jx&aW>-pFB`de*@blH|hs0yQ-7Q<~&{MN1O$cPn=2(G+a zb;!}CgGjRu=7bzDOhinK0V0>Y(5b3`3r*IYX`Pu~>r5Sn`QYn;&NA3kfp+EW@7tp8 zBQcX7cN-)FOQ}3c=O6rMG2i^;3`#)FJ~DToj|<%UZ#gRmr^O5LX@`w{o5igtsGwBu|T~?r%Uo&5Won^adpiJ&q9y zpHo%-$7kN&r?ysrcidZO)-$dZZ&$SaFzTI{d?0PO(*KxyX4JRD9X;|(T2HkUQVN#& zsyJDazdMp&2~MaCM&&u304l>r$aBOh@u$U+%U9He@DwtG%6{Q5j&Qh?hh*BEWIssc z6r#K)Mo6=~A`Bybgm~yqburNt3rk3!#5cpq_0|IRXzGDjvD#eiyMcjO%%uaLi}sYM z0O3&g+0{a(3ut4vOM8e^TY{G(N0@_1oz{d&WdHTldi7_Uanjdnj!8}&>S;Qu)EnEH z*o8q>3XLMH9sO>U2mi-#j6F5K%FPcSV}@=7$UK_p&ajgg7phQ2`_<#WCioK7|MTk; zN8}-pg-*{S5r5D;x7M>C4Ii}T)h}_#6gz33N7XExsQ(xy;zwwq1c{}g=Vw_$%zEyBK_ z-a7Nico&yVicP}T-+5jCCO8@l_LwkjTd_7G!sT@R%W?9NSX{V0diJbO@bdb)7eQBF z*(>59(-K0L)0Z1B&WffJciDR;V^+Ztiaq)}h&z7HoH;I_8cV*0J6@F+*?u#>oz3Oa zo(xnzwB*#%$;}*t_=uo0eGLuYE!Nb5RJILuWo_w9O9F{OGE#CEyC!u3B7>Pv46Q6J zQINF;64+j?=6R)E16(YxQOpV*_A(O@NoKa$jlRAl`sAE*em+t@v|94>3apgf-s<3=CGvEq@P3&kwlQbn=NuP4OkFcTDD^QamE0C@T zp-Z94X$db5hGLlN@1G&>FvCt0g)PpJ0C1VDV-gHK=X+8u|ELIu|GZp=z~?Y_3pHoLJ)|ZXmEkZ=JVIXmrp}pt3TWKn17;A z+=2A;H}{XUH@tSY>E8X2*hwT>Sy4^9)?vD7X^9>6Y<}f96p7Ljiqcc7G`}nTzgH3c zpS)bna++afNY6j3>-?)K?|8t2EkpZqlaaPY?%K7h<`3Su^f2I1KyS`DGA^k$N1h%5&hTpje+`?#!hi3A zub9eHOGYi!D8>^rn4cI>GUGPtDvG-2D8n2Iu9#36JfE5M3TYc5n;Owj*Br5!jN5>q zmf9P4thB^BICLcto`ZBrjqz&T)&2um5-NvdXcCZ|hj`JmrusxyR(HOFs1JbZ!s+`A z98Z;sUq_PJUQ%Mffy7qau3E^5uIb|PL_G@xBuYQR%5>ah^~?VXVDGGmc6Rj1h;I`A zjnq%kLFNHPQy0gQ1jR@JnPX1fL!KEc& Date: Sun, 14 Jan 2024 10:08:51 -0800 Subject: [PATCH 4/9] Fix failed precommit checks (#412) --- docs/tutorials/Fine-Tuning-Semantic-Parsing.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/tutorials/Fine-Tuning-Semantic-Parsing.md b/docs/tutorials/Fine-Tuning-Semantic-Parsing.md index 896649adc..c704867e4 100644 --- a/docs/tutorials/Fine-Tuning-Semantic-Parsing.md +++ b/docs/tutorials/Fine-Tuning-Semantic-Parsing.md @@ -1,10 +1,10 @@ # Fine-Tuning for Semantic Parsing -Semantic parsing is a process that transforms a natural language sentence into a logical form. -This logical form represents the sentence's meaning in a way that computer programs can utilize to carry out tasks, respond to questions, or follow commands. +Semantic parsing is a process that transforms a natural language sentence into a logical form. +This logical form represents the sentence's meaning in a way that computer programs can utilize to carry out tasks, respond to questions, or follow commands. -For example, through semantic parsing, a chatbot can convert a question posed in natural language into a precise SQL query, which then retrieves the desired information from a database. -Similarly, a virtual assistant can interpret a user's spoken request and, by employing semantic parsing, translate it into a JSON object that triggers specific actions. +For example, through semantic parsing, a chatbot can convert a question posed in natural language into a precise SQL query, which then retrieves the desired information from a database. +Similarly, a virtual assistant can interpret a user's spoken request and, by employing semantic parsing, translate it into a JSON object that triggers specific actions. By bridging the gap between human language and machine-readable formats, semantic parsing empowers AI systems to perform tasks with both accuracy and autonomy. In this post, we'll guide you through fine-tuning a [Llama2 model](https://ai.meta.com/llama/) for semantic parsing with Levanter. @@ -45,8 +45,8 @@ Expected Response: confirm(name[The Elder Scrolls Online], developer[ZeniMax Onl ### Quick Test with Llama2-7B Chat Model -If we test with the Llama2-7B chat model on these examples (see the full prompt in [Appendix A](#appendix-a-the-prompt-used-in-this-task)), -we can see it does not learn the task well: it struggles to generate the correct function names and hallucinates quite a few attributes that are not mentioned in the query; it also produces outputs with incorrect formats (`\n` before attribute names, `[E (`, etc). +If we test with the Llama2-7B chat model on these examples (see the full prompt in [Appendix A](#appendix-a-the-prompt-used-in-this-task)), +we can see it does not learn the task well: it struggles to generate the correct function names and hallucinates quite a few attributes that are not mentioned in the query; it also produces outputs with incorrect formats (`\n` before attribute names, `[E (`, etc). ``` Query: I had fun playing Age of Empires II: The Age of Kings. I enjoy a lot of multiplayer games from 1999. @@ -100,7 +100,7 @@ with open("train.jsonl", "w") as f: f.write("\n") ``` -The `PROMPT` provides the model with instructions to enhance its understanding of the task at hand. +The `PROMPT` provides the model with instructions to enhance its understanding of the task at hand. In our example, the prompt details the potential function names and attributes, aiding the model in generating the correct output. We provide the full prompt in [Appendix A](#appendix-a-the-prompt-used-in-this-task). While helpful, including a prompt is optional for fine-tuning. @@ -206,7 +206,7 @@ To save the LoRA adaptors as separate weight file, use `--hf_save_path` instead. ### Metrics How do we accurately evaluate a model's performance in semantic parsing tasks? -Character-level accuracy falls short as it doesn't account for variations in the order of attributes and does not distinguish between function names and attributes. +Character-level accuracy falls short as it doesn't account for variations in the order of attributes and does not distinguish between function names and attributes. Instead, we assess the model's ability to interpret instructions and parse semantic meaning from input queries by measuring more specific accuracies: - Function Name Accuracy: This metric confirms whether the extracted function name matches the expected one. @@ -424,4 +424,4 @@ Output: request_explanation(release_year[2005], rating[excellent]) Example 9) Sentence: Do you think Mac is a better gaming platform than others? Output: request_attribute(has_mac_release[]) -``` \ No newline at end of file +``` From adcc4216c296c92f265b03fc8b4a7b19a82e9dc4 Mon Sep 17 00:00:00 2001 From: Anh Tong Date: Thu, 18 Jan 2024 01:59:36 +0900 Subject: [PATCH 5/9] Add weight decay masking for optimizer (#405) * add weight decay masking * weight masking with regex * fix regex * refactor that build mask stays inside optimizer config --- src/levanter/trainer.py | 33 ++++++++++++++-- tests/test_weight_decay_mask.py | 67 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 tests/test_weight_decay_mask.py diff --git a/src/levanter/trainer.py b/src/levanter/trainer.py index fee40212c..c615dc1a3 100644 --- a/src/levanter/trainer.py +++ b/src/levanter/trainer.py @@ -3,6 +3,7 @@ import functools import logging as pylogging import os +import re import sys import typing import warnings @@ -38,7 +39,7 @@ from levanter.logging import WandbConfig, capture_time from levanter.types import FilterSpec from levanter.utils import cloud_utils -from levanter.utils.jax_utils import is_inexact_arrayish +from levanter.utils.jax_utils import is_inexact_arrayish, leaf_key_paths from levanter.utils.tree_utils import inference_mode @@ -707,9 +708,14 @@ class OptimizerConfig: cooldown: float = 0.0 """fraction of training steps to use as cooldown, or steps to use. 0.0 means no cooldown""" lr_schedule: str = "cosine" # constant, cosine, linear + """a regex or a list of strings to identify where to mask weight. """ + """For nano-GPT, this field can be set as + `r".*attn.*weight|.*mlp.*weight|.*token_embeddings|.*position_embeddings"`""" + weight_decay_modules: Optional[Union[List[str], str]] = None def build(self, num_train_steps: int) -> GradientTransformation: """Creates the optimizer""" + # indirection makes it work with optax.inject_hyperparams so we can log the learning rate def _optimizer(learning_rate): components = [] @@ -720,8 +726,7 @@ def _optimizer(learning_rate): components.append(optax.scale_by_adam(self.beta1, self.beta2, self.epsilon)) if self.weight_decay > 0: - # TODO: add weight decay masking?? - components.append(optax.add_decayed_weights(self.weight_decay)) + components.append(optax.add_decayed_weights(self.weight_decay, self.build_weight_decay_mask())) # - learning rate for descent components.append(optax.scale(-learning_rate)) @@ -732,6 +737,28 @@ def _optimizer(learning_rate): return optax.inject_hyperparams(_optimizer)(learning_rate=self.lr_scheduler(num_train_steps)) + def build_weight_decay_mask(self): + if self.weight_decay_modules is None: + return None + else: + # mask based on regex or module path + def _apply_on(x, key_path): + if isinstance(self.weight_decay_modules, str): + compiled_regex = re.compile(self.weight_decay_modules) + return compiled_regex.match(key_path) is not None + else: + return any(key_path.__contains__(target) for target in self.weight_decay_modules) + + def mask_fn(model): + return jax.tree_util.tree_map( + _apply_on, + model, + leaf_key_paths(model, is_leaf=eqx.is_array), + is_leaf=eqx.is_array, + ) + + return mask_fn + def lr_scheduler(self, num_train_steps): warmup_steps = self._convert_warmup(num_train_steps) cooldown_steps = _convert_ratio_or_steps(self.cooldown, num_train_steps) diff --git a/tests/test_weight_decay_mask.py b/tests/test_weight_decay_mask.py new file mode 100644 index 000000000..0c0f00e5c --- /dev/null +++ b/tests/test_weight_decay_mask.py @@ -0,0 +1,67 @@ +import equinox as eqx +import jax +import jax.random as jrandom + +import haliax as hax + +from levanter.models.gpt2 import Gpt2Config +from levanter.trainer import OptimizerConfig + + +def test_weight_decay_masking(): + def tree_at_mask(params): + # let's mask all leaves as False + params = jax.tree_util.tree_map(lambda _: False, params) + + def apply_weight_decay(tree): + # there is no weight decay performed in LayerNorms and bias + nodes = [] + + # apply on embedding + nodes.append(tree.embeddings.token_embeddings.array) + nodes.append(tree.embeddings.position_embeddings.array) + + # apply on attention + nodes.append(tree.transformer.blocks.stacked.attn.c_attn.weight.array) + nodes.append(tree.transformer.blocks.stacked.attn.c_proj.weight.array) + + # apply on MLP + nodes.append(tree.transformer.blocks.stacked.mlp.c_fc.weight.array) + nodes.append(tree.transformer.blocks.stacked.mlp.c_proj.weight.array) + + return nodes + + # apply weight decay when necessary + params = eqx.tree_at( + where=apply_weight_decay, + pytree=params, + replace_fn=lambda _: True, + ) + + return params + + gpt_config = Gpt2Config() + Vocab = hax.Axis("vocab", 100) + model = gpt_config.build(Vocab, key=jrandom.PRNGKey(0)) + string_list_config = OptimizerConfig( + weight_decay_modules=[ + "attn.c_attn.weight", + "attn.c_proj.weight", + "mlp.c_fc.weight", + "mlp.c_proj.weight", + "token_embeddings", + "position_embeddings", + ] + ) + regex_config = OptimizerConfig( + weight_decay_modules=r".*attn.*weight|.*mlp.*weight|.*token_embeddings|.*position_embeddings", + ) + # masking using `equinox.tree_at` + true_mask = tree_at_mask(model) + # masking using list of module path + list_string_mask = string_list_config.build_weight_decay_mask()(model) + + regex_mask = regex_config.build_weight_decay_mask()(model) + + assert eqx.tree_equal(list_string_mask, true_mask) + assert eqx.tree_equal(regex_mask, true_mask) From 9b2c11fe469cfd152718ef190e33c7b485bb772d Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 19 Jan 2024 21:46:07 -0800 Subject: [PATCH 6/9] bugfix for model parallelism (#423) --- src/levanter/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/levanter/trainer.py b/src/levanter/trainer.py index c615dc1a3..2a8c5e93c 100644 --- a/src/levanter/trainer.py +++ b/src/levanter/trainer.py @@ -678,7 +678,7 @@ def _validate_and_set_defaults(self): raise ValueError("either model_axis_size or local_device_count must be divisible by the other") if self.per_device_parallelism == -1: - self.per_device_parallelism = self.train_batch_size // jax.device_count() + self.per_device_parallelism = self.train_batch_size // self.data_axis_size # validate size of per_device_parallelism if self.train_batch_size % (self.per_device_parallelism * self.data_axis_size) != 0: From e27d39ca4519cd54872672277c2759da4aefa81a Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 22 Jan 2024 15:41:15 -0800 Subject: [PATCH 7/9] Alpaca tutorial and qol improvements (#425) * make the prompts in alpaca.yaml explicit * accept compressed json * update docs for alpaca --- docs/Fine-Tuning.md | 385 ++++++++++++++++------------- examples/alpaca/alpaca-llama2.yaml | 19 ++ examples/alpaca/alpaca.py | 11 +- examples/alpaca/alpaca.yaml | 19 ++ 4 files changed, 260 insertions(+), 174 deletions(-) diff --git a/docs/Fine-Tuning.md b/docs/Fine-Tuning.md index 4903e3c1e..58e8f455d 100644 --- a/docs/Fine-Tuning.md +++ b/docs/Fine-Tuning.md @@ -2,31 +2,35 @@ While Levanter's main focus is pretraining, we can also use it for fine-tuning. As an example, we'll show how to reproduce [Stanford Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html), -using [Levanter](https://github.com/stanford-crfm/levanter) and either Llama 1 or [Llama 2](https://ai.meta.com/llama/). +using [Levanter](https://github.com/stanford-crfm/levanter) and either [Llama 1](https://arxiv.org/abs/2302.13971) or [Llama 2](https://ai.meta.com/llama/) 7B. The script we develop will be designed for Alpaca, defaulting to using its dataset and prompts, but it should work for any single-turn instruction-following task. -This tutorial is meant to cover "full finetuning", where you start with a pretrained model and modify -all of its parameters to fit some final task, rather than something like LoRA (though see our [LoRA tutorial](./LoRA.md) for that). -It also documents how to work with datasets that aren't just single `"text"`s, as we use in pretraining. +This tutorial is meant to cover "full finetuning," where you start with a pretrained model and modify +all of its parameters to fit some final task, rather than something like LoRA that adds a (small) number +of additional parameters. (See our [LoRA tutorial](./LoRA.md) for that.) +It also documents how to work with datasets that aren't just single `"text"`s, which is what we use in pretraining. ## Overview of Alpaca -[Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html) is a lightweight fine tune of Llama 1 on a +[Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html) is a fine tune of Llama 1 on a [dataset of 52000 input/output pairs](https://huggingface.co/datasets/tatsu-lab/alpaca), which were generated by taking [a seed set from self-instruct](https://github.com/yizhongw/self-instruct) and asking `text-davinci-003` to generate more examples. +The original Alpaca script is [here](https://github.com/tatsu-lab/stanford_alpaca/blob/main/train.py). + ![Schematic diagram of how the Alpaca model was created](https://crfm.stanford.edu/static/img/posts/2023-03-13-alpaca/alpaca_main.jpg) ### The Foundation Model -Llama 1 is a 7B parameter causal language model trained on 1T tokens from various mostly English sources. It's described -in [the Llama paper](https://arxiv.org/abs/2302.13971). +Llama 1 7B is a ≈7 billion parameter causal language model trained on 1 trillion tokens from various mostly English sources. +It's described in [the Llama paper](https://arxiv.org/abs/2302.13971). [Llama 2](https://ai.meta.com/llama/) is a similar model, +just trained on more data (and with some slight tweaks to the architecture for larger models). ### The Data -More precisely, the dataset is composed of triples of (instruction, input, output), where the instruction is a prompt +The Alpaca dataset is composed of triples of (instruction, input, output), where the instruction is a prompt describing the task. A bit less than 40% of the examples have inputs, and the rest are just the instruction and output. Here are some example inputs, instructions, and outputs: @@ -44,7 +48,8 @@ generated by an LLM after all.) But it's a good example of the kind of data you ### Preprocessing Because Llama is a causal language model, we need to do some preprocessing to turn the pairs/triples into -a single sequence. The usual thing is to interpolate the strings into a prompt. We'll have two prompts, +a single sequence. The usual thing is to interpolate the strings into a prompt that provides +some context/guidance to the LM. We'll have two prompts, depending on whether or not there's an input or just an instruction and output. For example, the first example above would be turned into: @@ -74,10 +79,11 @@ Compute the area of a rectangle with length 10cm and width 5cm. The area of the rectangle is 50 cm2. ``` -From there [original Alpaca script](https://github.com/tatsu-lab/stanford_alpaca/blob/main/train.py) *masks out the loss* for all tokens before the start of the output. This gets -the model to learn to mimic outputs conditioned on inputs, rather than spending time learning to generate inputs and outputs. +From there, [the original Alpaca script](https://github.com/tatsu-lab/stanford_alpaca/blob/main/train.py) *masks out the loss* for all tokens before the start of the output. This gets +the model to learn to mimic outputs conditioned on inputs, rather than getting the model to learn the prompts and +inputs along with the outputs. -## Setup +## Running the script Rather than going through the code first, we'll jump straight to running the script. We'll cover the code in the [Code Walkthrough](#code-walkthrough) section below. @@ -86,61 +92,170 @@ Rather than going through the code first, we'll jump straight to running the scr Make sure you go through either the [GPU](./Getting-Started-GPU.md) or [TPU](./Getting-Started-TPU-VM.md) setup, depending on what you want to use. -### Environment Setup +### NVIDIA GPU -#### \[GPU\] Environment Setup +#### Environment Setup Follow the instructions in the [Getting Started with GPUs](./Getting-Started-GPU.md) guide to create a conda environment or virtualenv -and to install JAX with CUDA. +and to install JAX with CUDA. Then, if you haven't already, clone the Levanter repo and install it in editable mode: + +```bash +git clone https://github.com/stanford-crfm/levanter.git +cd levanter +pip install -e . +``` + +You'll also want to log into [WANDB](https://wandb.ai/). + +```bash +wandb login +``` + +To use Llama 2, you'll need to request access to the model from [Llama 2's Hugging Face page](https://huggingface.co/meta-llama/Llama-2-7b-hf). +Then, you'll need to log into the Hugging Face CLI: + +```bash +huggingface-cli login +``` + +#### Running the Script + +The example commands below demonstrate how to launch a training job on a node with 8 A100 GPUs, but should work for +other single node GPU configurations. For example, we've also tested Alpaca replication with +a node of 8 RTX 6000 Ada Generation 49.1GB GPUs. Levanter works best with Ada or later generation NVIDIA GPUs. +To replicate Alpaca, you can run the following command: + +```bash +python examples/alpaca/alpaca.py --config_path levanter/examples/alpaca/alpaca.yaml +``` + +To use Llama 2: + +```bash +python examples/alpaca/alpaca.py --config_path levanter/examples/alpaca/alpaca-llama2.yaml +``` + +Alternatively: + +```bash +python examples/alpaca/alpaca.py --config_path levanter/examples/alpaca/alpaca-llama2.yaml --model_name_or_path meta-llama/Llama-2-7b-hf +``` + +!!! warning + + Fine-tuning a 7B parameter model needs **a lot** of accelerator memory, you will need more than 80GB of GPU memory in + aggregate to run this job. Because Levanter makes heavy use of FSDP, you can use several smaller cards. + If you don't have enough memory, you can try reducing the `train_batch_size` or the `per_device_parallelism` in + the config. + + +At some point the run will spit out a WandB link. You can click on that to see the training progress. There's +not a ton to see there (yet), but you can see the training loss go down over time. + +On an 8xA100 box, training should take about ~3.5 hours, similar to the original Alpaca script. +It should take ~8.5 hours on 8 RTX 6000 Ada Generation GPUs. + + +### TPUs -#### \[TPU\] Environment Setup +#### Environment Setup -For TPUs, please follow the instructions in the [Getting Started with TPUs](./Getting-Started-TPU-VM.md) guide to -get started with a TPU VM. Once you have, you can just run the -following command from a source checkout of Levanter: +For TPUs, please follow the instructions in the [Getting Started with TPUs](./Getting-Started-TPU-VM.md). +Once you have, you can run something like this to get a v3-32 TPU VM: ```bash bash infra/spin-up-vm.sh llama-32 -z us-east1-d -t v3-32 --preemptible ``` -### Install Levanter +You might need to change the zone and/or the TPU type depending on what's available. You can also use preemptible +TPUs if you want to save money (or that's what your quota is). I think training Alpaca should work on a v3-8, +but we don't have any of those. + +#### Running the Script + +Launching the run on TPU is a bit more complex because you need to specify a lot of paths to GCS buckets. You will +also likely need to run the command on multiple machines, because a v3-32 VM is actually 4 distinct machines, each +controlling 8 TPUs. + +This is what the command looks like: ```bash +export GCS_BASE="gs://" +gcloud compute tpus tpu-vm ssh llama-32 --zone us-east1-d --worker=all \ +--command="WANDB_API_KEY=${YOUR TOKEN HERE} \ +HUGGING_FACE_HUB_TOKEN=${YOUR TOKEN HERE} \ +bash levanter/infra/run.sh python \ +levanter/examples/alpaca/alpaca.py \ +--config_path levanter/examples/alpaca/alpaca-llama2.yaml \ +--data_cache_dir ${GCS_BASE}/data \ +--trainer.checkpointer.base_path ${GCS_BASE}/ckpts \ +--hf_save_path ${GCS_BASE}/hf_ckpts +``` + +If you're using preemptible or TRC TPUs, you'll want to add `--trainer.id ` to the command line. +Alternatively, you can use the [babysitting script](./Getting-Started-TPU-VM.md#babysitting-script) to automatically restart the +VM and job if it gets preempted. (It will also set a run id automatically.) That would look like this: ```bash -git clone https://github.com/stanford-crfm/levanter.git -cd levanter -pip install -e . +infra/babysit-tpu-vm.sh llama-32 -z us-east1-d -t v3-32 --preemptible -- \ +WANDB_API_KEY=${YOUR TOKEN HERE} \ +HUGGING_FACE_HUB_TOKEN=${YOUR TOKEN HERE} \ +bash levanter/infra/run.sh python \ +levanter/examples/alpaca/alpaca.py \ +--config_path levanter/examples/alpaca/alpaca-llama2.yaml \ +--trainer.checkpointer.base_path gs:// \ +--hf_save_path gs:// \ ``` +You should see a link to the WandB run in the output. You can click on that to see the training progress. +Similar to an 8xA100 box, training should take about ~3.5 hours on a v3-32. + + ## Configuration -We have two configs for Alpaca: one for Llama 1 and one for Llama 2. The only difference is the model id. +That should be all you need to run the script and replicate Alpaca. +However, if you want to customize the script, you can do so by modifying the config. +We have two configs for Alpaca: one for Llama 1 and one for Llama 2. The only difference is the `model_name_or_path` field. -### Config to Replicate Alpaca +### Base Config ```yaml # cf https://github.com/tatsu-lab/stanford_alpaca#fine-tuning data: tatsu-lab/alpaca model_name_or_path: huggyllama/llama-7b trainer: - mp: p=f32,c=bfloat16 + mp: p=f32,c=bfloat16 # Mixed precision training with fp32 parameters/optimizer state and bf16 activations wandb: project: "levanter-alpaca" num_train_steps: 1218 # 128 * 1218 = 155904, which is almost but not quite 3 epochs, which is what alpaca did train_batch_size: 128 - # if using model parallelism, this is useful: - tensor_parallel_axes: ["mlp", "heads"] optimizer: learning_rate: 2e-5 weight_decay: 0.0 +prompts: + # |- means multiline string, keeping all but the final newline + prompt_input: |- + Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Input: + {input} + + ### Response: + prompt_no_input: |- + Below is an instruction that describes a task. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Response: ``` This config uses mixed fp32/bf16 precision and sets the number of training steps to be roughly 3 epochs. It sets up the optimizer to use a learning rate of 2e-5 and no weight decay. `trainer.per_device_parallelism` is roughly equivalent to HF's -`per_device_train_batch_size`. If you want to use model parallelism, you can set `trainer.model_axis_size` to something -like 2. (This will split the model across two devices. This might be useful if you're using a v3-64 or something similar and -want to maintain the same batch size.) +`per_device_train_batch_size`. ### Llama 2 Config @@ -150,156 +265,79 @@ If you haven't already, go to [Llama 2's Hugging Face page](https://huggingface. Once you have access, go to [Hugging Face's Tokens page](https://huggingface.co/settings/tokens) to get an API token. You'll need to provide this to the TPU VM as an environment variable. (We'll show you how to do this later.) -### Customizing the Config - -If you have your own dataset, you'll want to change the `data` field in the config to point to your dataset. -You'll also want to change the `model_name_or_path` field to point to the model you want to use. -Currently, Levanter supports GPT-2, Llama, MPT, and Backpack checkpoints. - -```yaml -data: # Path to the training data, or huggingface dataset name. -data_cache_dir: -model_name_or_path: "meta-llama/Llama-2-7b-chat-hf" -trainer: - ... -``` - -#### Custom Prompts - -If you want to use your own prompts, you can add them to the config, like this: - -```yaml -prompts: - prompt_input: |- - ### Instruction: {instruction} - ### Input: {input} - ### Output: - prompt_no_input: |- - ### Instruction: {instruction} - ### Output: -``` - -#### Custom Datasets +### Custom Datasets -The script we develop in this tutorial is designed for Alpaca, but it should work for any single-turn -instruction-following task. For instance, to train a code alpaca model, you could use the following config: +The script in this tutorial is designed for Alpaca, but it should work for any single-turn +instruction-following task. For instance, to train a Code Alpaca model, you could modify the config like this: ```yaml data: lucasmccabe-lmi/CodeAlpaca-20k # a dataset on the Hugging Face hub data_cache_dir: code_alpaca_cache # some path to store the cache ``` -The dataset can also be a JSON path. +The dataset can also be a path to a JSON or JSONL file, or compressed versions of those. -#### \[TPU\] Using a Modified Config +### Custom Models -If you make changes to the config, you'll need to get the config file to all the workers. For TPU, the best way to do this -is to copy it to Google Cloud Storage so that it persists when the machine is preempted. You can do this with: +You can also change the `model_name_or_path` field to point to the model you want to use. This +can be any Hugging Face model, or a path to a local checkpoint. Currently, Levanter supports GPT-2, Llama, MPT, and +Backpack checkpoints. -```bash -gsutil cp examples/alpaca/alpaca.yaml gs:///train-alpaca.yaml +```yaml +model_name_or_path: "meta-llama/Llama-2-7b-chat-hf" ``` -If using Llama 2: +Or on the command line: ```bash -gsutil cp examples/alpaca/alpaca-llama2.yaml gs:///train-alpaca.yaml +python examples/alpaca/alpaca.py --config_path levanter/examples/alpaca/alpaca.yaml --model_name_or_path "meta-llama/Llama-2-7b-chat-hf" ``` -And then using `--config_path gs:///alpaca.yaml` instead of `--config_path levanter/examples/alpaca/train-alpaca.yaml` -in the command line below. Levanter knows how to read from Google Cloud Storage, so you don't need to do anything else. +### Custom Prompts -## Launching the Job +If you want to use your own prompts, you can modify the `prompts` field. By default, the prompts are set to be the same +as the original Alpaca, but you can change them to whatever you want. They are formatted using Python's +[format strings](https://docs.python.org/3/library/string.html#format-string-syntax), meaning you can use `{instruction}` and `{input}`. +You should have two prompts: one for when there's an input and one for when there isn't. For example, here is +a more minimal prompt: -### \[GPU\] Launching the Job - -Right now, Levanter is only known to work with single node GPU training. The example commands below demonstrate how to launch a training job -on a node with 8 A100 GPUs, but should work for other single node GPU configurations. For example, we've also tested Alpaca replication with -a node of RTX 6000 Ada Generation 49.1GB GPUs. - -Before running your training bash command, ensure you are in your `levanter` conda environment, you've created a directory for saving checkpoints -during training, you are logged into your wandb account with the following two commands: - -!!! warning - - Fine-tuning a 7B parameter model needs **a lot** of accelerator memory, you will need more than 80GB of GPU memory in - aggregate to run this job. Because Levanter makes heavy use of FSDP, you can use several smaller cards. - If you don't have enough memory, you can try reducing the `train_batch_size` or the `per_device_parallelism` in - the config. - - - -```bash -conda activate levanter -wandb login ${YOUR TOKEN HERE} +```yaml +prompts: + prompt_input: |- + ### Instruction: {instruction} + ### Input: {input} + ### Output: + prompt_no_input: |- + ### Instruction: {instruction} + ### Output: ``` -Now you can run the training command: -```bash -python examples/alpaca/alpaca/alpaca.py \ ---config_path examples/alpaca/alpaca.yaml \ ---trainer.checkpointer.base_path levanter/checkpoints \ ---hf_save_path levanter/checkpoints -``` +We use YAML's [multiline string syntax](https://yaml-multiline.info/) to make the prompts easier to read. +You can also specify a path to a json file containing the prompts if you'd prefer. + -You can change `--trainer.checkpointer.base_path` and `--hf_save_path` to your desired model checkpoint directories. +### \[TPU\] Using a Modified Config -If you're using Llama 2, you'll need to first request access to the model, and then export your Hugging Face API token: +On a single machine, you can just modify the config and run the script. On TPU, however, you'll need to upload the config +to a Google Cloud Storage bucket so that all the workers can access it. You can do this with: ```bash -export HUGGING_FACE_HUB_TOKEN=${YOUR TOKEN HERE} +gsutil cp my-config.yaml gs:///my-config.yaml ``` -or log into the HF CLI with `huggingface-cli login`. +And then using `--config_path gs:///my-config` instead of `--config_path levanter/examples/alpaca/train-alpaca.yaml` +in the command line. Levanter knows how to read from Google Cloud Storage, so you don't need to do anything else. -### \[GPU\] NLP-Group Slurm Cluster Launch Example +### Aside: Running on Slurm -Say you save the above Alpaca training command as a bash script called `train_alpaca.sh`, then +Say you save the above Alpaca training command as a bash script called `train_alpaca.sh`. Then you could launch a training job on a slurm cluster with `srun` as follows: ```bash srun --account=nlp --cpus-per-task=32 --gpus-per-node=8 --mem=400G --open-mode=append --partition=sphinx --nodes=1 --pty bash train_alpaca.sh ``` -### \[TPU\] Launching the Job - -For TPU, we need just a little bit of ceremony to get the Hugging Face and WANDB API tokens in the environment: -(If you're using Llama 1, you don't need the `HUGGING_FACE_HUB_TOKEN` line unless you're using a private model -or uploading to Hugging Face.) - -```bash -gcloud compute tpus tpu-vm ssh llama-32 --zone us-east1-d --worker=all \ ---command="WANDB_API_KEY=${YOUR TOKEN HERE} \ -HUGGING_FACE_HUB_TOKEN=${YOUR TOKEN HERE} \ -bash levanter/infra/run.sh python \ -levanter/examples/alpaca/alpaca.py \ ---config_path levanter/examples/alpaca/alpaca.yaml \ ---trainer.checkpointer.base_path gs:// \ ---hf_save_path gs:// -``` - -If you're using preemptible or TRC TPUs, you'll want to add `--trainer.id ` to the command line. -Alternatively, you can use the [babysitting script](./Getting-Started-TPU-VM.md#babysitting-script) to automatically restart the -VM and job if it gets preempted. That would look like this: - -```bash -infra/babysit-tpu-vm.sh llama-32 -z us-east1-d -t v3-32 --preemptible -- \ -WANDB_API_KEY=${YOUR TOKEN HERE} \ -HUGGING_FACE_HUB_TOKEN=${YOUR TOKEN HERE} \ -bash levanter/infra/run.sh python \ -levanter/examples/alpaca/alpaca.py \ ---config_path levanter/examples/alpaca/alpaca-llama2.yaml \ ---trainer.checkpointer.base_path gs:// \ ---hf_save_path gs:// \ -``` - -## Waiting - -At some point the run will spit out a WandB link. You can click on that to see the training progress. There's -not a ton to see there (yet), but you can see the training loss go down over time. - -On a v3-32 or an 8xA100 box, training should take about ~3.5 hours, similarly on a v3-32 TPU VM. -It should take ~8.5 hours on 8 RTX 6000 Ada Generation GPUs. +(This is for the Stanford NLP Cluster. Adjust as necessary for your cluster.) ## Using the Model @@ -309,7 +347,7 @@ When you're done, you can copy out the Hugging Face model with: gsutil cp -r gs:////step- ./my-alpaca ``` -The model should work out-of-the-box as a Hugging Face model. You can use it like this: +The model should work out-of-the-box as a Hugging Face model. For a quick test, you can use it like this: ```python from transformers import AutoModelForCausalLM, AutoTokenizer @@ -344,7 +382,7 @@ If you want to just run the script, you can skip to the [Setup](#setup) section. ### Approach Levanter's existing main entry points are designed for "pure" causal language modeling, where you have a single sequence -and don't want to mask out any tokens. So we'll instead write a custom script that does the following: +and don't have any prompts or custom formatting. So we'll instead write a custom script that does the following: * Preprocesses the dataset into a single sequence, interpolating prompts as we go. We'll also construct a `loss_mask` and do any padding. * Loads the model and resizes the vocabulary to match the tokenizer. @@ -359,7 +397,7 @@ if you want more information. The first step is to get the dataset. We'll use the [Hugging Face Dataset version](https://huggingface.co/datasets/tatsu-lab/alpaca) to do this. (You can also download it directly from the [dataset page](https://huggingface.co/datasets/tatsu-lab/alpaca), but -Levanter's integration with Hugging Face datasets makes it a bit easier to use.) +Levanter's integration with Hugging Face datasets means we don't need to do that.) ```python def _get_data_source(path_or_id): @@ -371,8 +409,8 @@ def _get_data_source(path_or_id): return levanter.data.dataset_from_hf(path_or_id, split="train") ``` -Preprocessing in Levanter typically comes in two phases: -* creating the on-disk cache, +Preprocessing in Levanter typically happens in two phases: +* creating an on-disk cache of the "heavy" preprocessing, like tokenization; and * transforming examples from the cache into the examples that the model expects. Here's the first phase, where we create the cache. We basically want to interpolate the prompt with the input @@ -380,19 +418,19 @@ and instructions, and then tokenize the result. We also want to keep track of th can mask out the loss appropriately. ```python -def mk_dataset(data_path_or_id: str, cache_dir: str, tokenizer): - # wrap an HF dataset with Levanter's native dataset class for fancier preprocessing. - # Levanter's native dataset class supports streaming, deteriministic, distributed preprocessing out of the box, - # which is a bit overkill for this dataset, but it's a good example of how to use it. - dataset = _get_data_source(data_path_or_id) +def mk_dataset(config: TrainArgs, tokenizer: transformers.PreTrainedTokenizerBase): + dataset = _get_data_source(config.data) - prompt_input, prompt_no_input = PROMPT_DICT["prompt_input"], PROMPT_DICT["prompt_no_input"] + prompts = get_prompts(config.prompts) def preprocess(batch): - sources = [ - prompt_input.format_map(example) if example.get("input", "") != "" else prompt_no_input.format_map(example) - for example in batch - ] + def format_example(ex): + if ex.get("input", "") == "": + return prompts["prompt_no_input"].format_map(ex) + else: + return prompts["prompt_input"].format_map(ex) + + sources = [format_example(example) for example in batch] targets = [f"{example['output']}{tokenizer.eos_token}" for example in batch] # TODO: this seems pretty wasteful since you end up tokenizing twice, but it's how the original code does it. examples = [s + t for s, t in zip(sources, targets)] @@ -407,16 +445,15 @@ def mk_dataset(data_path_or_id: str, cache_dir: str, tokenizer): } dataset = dataset.map_batches(preprocess, batch_size=128, num_cpus=num_cpus_used_by_tokenizer(tokenizer)) - dataset = dataset.build_cache(cache_dir, await_finished=True) + dataset = dataset.build_cache(config.data_cache_dir, await_finished=True) - # SupervisedDataset does last minute padding and masking - dataset = SupervisedDataset(dataset, tokenizer) + dataset = SupervisedDataset(dataset, tokenizer, mask_inputs=config.mask_inputs) return dataset ``` -In the second, we create [levanter.models.lm.LmExample][] objects from the cache. These are the inputs to the model. -`LmExample`s look like this: +`SupervisedDataset` is a class that we'll define later that does the final transformation from the cache to the +`LmExample` objects that the model expects. `LmExample`s look like this: ```python class LmExample(eqx.Module): @@ -425,8 +462,9 @@ class LmExample(eqx.Module): attn_mask: AttentionMask = AttentionMask.causal() ``` -So we need to populate the first two fields. We'll do that with a dataset whose job is to take the cache and turn it into -`LmExample`s. +So we need to populate the first two fields. `tokens` is the input sequence, and `loss_mask` is a boolean mask +that tells the model which tokens to compute the loss for. (We mask out the loss for everything before the start +of the output.) ```python class SupervisedDataset(Dataset[LmExample]): @@ -458,3 +496,12 @@ class SupervisedDataset(Dataset[LmExample]): The rest is boilerplate: setting up the model, optimizer, and trainer, and then running the training loop. We'll skip over that in this tutorial, but you can see the full script [here](https://github.com/stanford-crfm/levanter/blob/main/examples/alpaca/alpaca.py) if you want to see how it works. + + +## Conclusion + +That's it for this tutorial: you should now be able to fine-tune Llama 1 or Llama 2 on Alpaca or any other single-turn +instruction-following task. If you want to learn more about Levanter, check out the [Levanter docs](https://levanter.readthedocs.io/en/latest/) +or the [Levanter repo](https://github.com/stanford-crfm/levanter). For discussion, you can find us on [Discord](https://discord.gg/CKazXcbbBm). + +Let us know what you'd like to see next! diff --git a/examples/alpaca/alpaca-llama2.yaml b/examples/alpaca/alpaca-llama2.yaml index 88ea6c917..2527de03f 100644 --- a/examples/alpaca/alpaca-llama2.yaml +++ b/examples/alpaca/alpaca-llama2.yaml @@ -13,3 +13,22 @@ trainer: optimizer: learning_rate: 2e-5 weight_decay: 0.0 +prompts: + # |- means multiline string, keeping all but the final newline + prompt_input: |- + Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Input: + {input} + + ### Response: + prompt_no_input: |- + Below is an instruction that describes a task. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Response: diff --git a/examples/alpaca/alpaca.py b/examples/alpaca/alpaca.py index 73d147ac6..e02b0738a 100644 --- a/examples/alpaca/alpaca.py +++ b/examples/alpaca/alpaca.py @@ -35,6 +35,7 @@ # Ways this script could be improved: # * Could tune hparams more for throughput +# Original # Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -93,7 +94,7 @@ class TrainArgs: model_cache_dir: Optional[str] = None # Path to cache the model. must be local. - hf_save_path: Optional[str] = None # Path to save the HuggingFace checkpoint. + hf_save_path: Optional[str] = "alpaca_hf_ckpts" # Path to save the HuggingFace checkpoint, can be gcs hf_upload: Union[bool, str] = False # Name of the HuggingFace repo to upload to (if any). hf_save_steps: int = 1000 # How often to save the HuggingFace checkpoint. @@ -134,14 +135,14 @@ def _get_data_source(path_or_id): """The original alpaca.py used a json file, but it's since been moved to the HF dataset hub. You can use any dataset that's compatible with the structure of the alpaca dataset.""" if fsspec_utils.exists(path_or_id): - # get file format: jsonl or json - if path_or_id.endswith(".jsonl"): + # we're a bit generous here b/c we support compression + if ".jsonl" in path_or_id: return JsonlDataset([path_or_id]) - elif path_or_id.endswith(".json"): + elif ".json" in path_or_id: return JsonDataset([path_or_id]) else: raise ValueError( - f"We only support HF Dataset or a data file with .json or .jsonl extensions, not {path_or_id}!" + f"We only support HF Datasets or a data file with .json or .jsonl extensions, not {path_or_id}!" ) else: return WrappedHFDataset(path_or_id, split="train") diff --git a/examples/alpaca/alpaca.yaml b/examples/alpaca/alpaca.yaml index 3eae596fc..fd0e2fb7c 100644 --- a/examples/alpaca/alpaca.yaml +++ b/examples/alpaca/alpaca.yaml @@ -13,3 +13,22 @@ trainer: optimizer: learning_rate: 2e-5 weight_decay: 0.0 +prompts: + # |- means multiline string, keeping all but the final newline + prompt_input: |- + Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Input: + {input} + + ### Response: + prompt_no_input: |- + Below is an instruction that describes a task. Write a response that appropriately completes the request. + + ### Instruction: + {instruction} + + ### Response: From 8250b776c28f2d929c4c095dc571b12f128ec11d Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 23 Jan 2024 11:26:54 -0800 Subject: [PATCH 8/9] Alpaca qol (#430) * make the prompts in alpaca.yaml explicit * accept compressed json * update docs for alpaca * misc typos --- docs/Fine-Tuning.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/Fine-Tuning.md b/docs/Fine-Tuning.md index 58e8f455d..0bd3545b0 100644 --- a/docs/Fine-Tuning.md +++ b/docs/Fine-Tuning.md @@ -13,8 +13,7 @@ It also documents how to work with datasets that aren't just single `"text"`s, w ## Overview of Alpaca -[Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html) is a fine tune of Llama 1 on a -[dataset of 52000 input/output pairs](https://huggingface.co/datasets/tatsu-lab/alpaca), which +[Alpaca](https://crfm.stanford.edu/2023/03/13/alpaca.html) is a fine tune of Llama 1 on a [dataset of 52000 input/output pairs](https://huggingface.co/datasets/tatsu-lab/alpaca), which were generated by taking [a seed set from self-instruct](https://github.com/yizhongw/self-instruct) and asking `text-davinci-003` to generate more examples. @@ -25,7 +24,7 @@ The original Alpaca script is [here](https://github.com/tatsu-lab/stanford_alpac ### The Foundation Model Llama 1 7B is a ≈7 billion parameter causal language model trained on 1 trillion tokens from various mostly English sources. -It's described in [the Llama paper](https://arxiv.org/abs/2302.13971). [Llama 2](https://ai.meta.com/llama/) is a similar model, +It's described in [the Llama 1 paper](https://arxiv.org/abs/2302.13971). [Llama 2](https://ai.meta.com/llama/) is a similar model, just trained on more data (and with some slight tweaks to the architecture for larger models). ### The Data @@ -97,7 +96,7 @@ Rather than going through the code first, we'll jump straight to running the scr #### Environment Setup Follow the instructions in the [Getting Started with GPUs](./Getting-Started-GPU.md) guide to create a conda environment or virtualenv -and to install JAX with CUDA. Then, if you haven't already, clone the Levanter repo and install it in editable mode: +and to install JAX with CUDA. Then, if you haven't already done so, clone the Levanter repository and install it in editable mode: ```bash git clone https://github.com/stanford-crfm/levanter.git @@ -122,7 +121,8 @@ huggingface-cli login The example commands below demonstrate how to launch a training job on a node with 8 A100 GPUs, but should work for other single node GPU configurations. For example, we've also tested Alpaca replication with -a node of 8 RTX 6000 Ada Generation 49.1GB GPUs. Levanter works best with Ada or later generation NVIDIA GPUs. +a node of 8 RTX 6000 Ada Generation 49.1GB GPUs. (Levanter works best with Ada or later generation NVIDIA GPUs.) + To replicate Alpaca, you can run the following command: ```bash @@ -143,7 +143,7 @@ python examples/alpaca/alpaca.py --config_path levanter/examples/alpaca/alpaca-l !!! warning - Fine-tuning a 7B parameter model needs **a lot** of accelerator memory, you will need more than 80GB of GPU memory in + Fine-tuning a 7B parameter model needs **a lot** of accelerator memory: you will need more than 80GB of GPU memory in aggregate to run this job. Because Levanter makes heavy use of FSDP, you can use several smaller cards. If you don't have enough memory, you can try reducing the `train_batch_size` or the `per_device_parallelism` in the config. @@ -168,7 +168,7 @@ bash infra/spin-up-vm.sh llama-32 -z us-east1-d -t v3-32 --preemptible ``` You might need to change the zone and/or the TPU type depending on what's available. You can also use preemptible -TPUs if you want to save money (or that's what your quota is). I think training Alpaca should work on a v3-8, +TPUs if you want to save money (or that's what your quota is). Training Alpaca should work on a v3-8, but we don't have any of those. #### Running the Script @@ -341,7 +341,7 @@ srun --account=nlp --cpus-per-task=32 --gpus-per-node=8 --mem=400G --open-mode=a ## Using the Model -When you're done, you can copy out the Hugging Face model with: +When you're done, you can download the Hugging Face model with: ```bash gsutil cp -r gs:////step- ./my-alpaca From efef064e5529f465c26aac1b0e79de56bc3dad98 Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 24 Jan 2024 22:27:18 -0800 Subject: [PATCH 9/9] Tweaks to improve multiprocess gpu outside of slurm (#431) --- pyproject.toml | 2 +- src/levanter/checkpoint.py | 4 +- src/levanter/distributed.py | 92 +++++++++++++++++++++++++--------- src/levanter/utils/py_utils.py | 2 +- tests/test_checkpoint.py | 3 +- 5 files changed, 74 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e3d8be709..96272c628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dependencies = [ "ray[default]", "pydantic<2", # temporary pin until Ray supports pydantic 2.0 "rich>=13", -# "chex>=0.1.85" + "filelock", ] [tool.hatch.build] diff --git a/src/levanter/checkpoint.py b/src/levanter/checkpoint.py index 70087af75..15f16a203 100644 --- a/src/levanter/checkpoint.py +++ b/src/levanter/checkpoint.py @@ -231,7 +231,7 @@ def save_checkpoint(self, info, destination: str): logger.info(f"Saved checkpoint at step {info.step} to {path}. Save time is {self._last_save_time}") -def save_checkpoint(model, training_state, step: int, checkpoint_path: PathLike, *, exist_ok: bool = False): +def save_checkpoint(model, training_state, step: int, checkpoint_path: PathLike): """ Save a checkpoint to a given path using TensorStore. If exist_ok is True, the checkpoint will be saved even if a checkpoint already exists at the given path. @@ -247,7 +247,7 @@ def save_checkpoint(model, training_state, step: int, checkpoint_path: PathLike, fs: AbstractFileSystem fs, plain_path = _get_fs_and_plain_path(checkpoint_path) - fs.makedirs(plain_path, exist_ok=exist_ok) + fs.makedirs(plain_path, exist_ok=True) tree_serialize_leaves_tensorstore(os.path.join(checkpoint_path, "model"), model) if training_state is not None: diff --git a/src/levanter/distributed.py b/src/levanter/distributed.py index f4a86f8b9..c0442b45e 100644 --- a/src/levanter/distributed.py +++ b/src/levanter/distributed.py @@ -224,29 +224,30 @@ def _munge_address_port(address: str): # this is no longer the case, so instead we need to check if we are the coordinator # and if so, start the head - if _is_this_machine(host): - logger.info(f"Starting ray head on port {ray_port}. We are process the coordinator {host}.") - logger.info(f"Starting ray with num_cpus set to {num_cpus}.") - ret = os.system( - f"ray start --head --port {ray_port} --num-cpus {num_cpus} --dashboard-host=0.0.0.0" - ) - if ret != 0: - raise RuntimeError(f"Failed to start ray head with exit code {ret}") - else: - logger.info(f"Successfully started ray head on port {ray_port}.") - - # install an atexit handler to kill the head when we exit - atexit.register(lambda: os.system("ray stop -g 10 --force")) - elif start_workers: - logger.info( - f"Starting ray worker and connecting to {address}. We are process {jax.process_index()}." - ) - logger.info(f"Starting ray with num_cpus set to {num_cpus}.") - ret = os.system(f"ray start --address {address} --num-cpus {num_cpus}") - if ret != 0: - raise RuntimeError(f"Failed to start ray head with exit code {ret}") - else: - logger.info(f"Successfully started ray worker and connected to {address}.") + if _is_local_leader(): + if _is_this_machine(host): + logger.info(f"Starting ray head on port {ray_port}. We are process the coordinator {host}.") + logger.info(f"Starting ray head with num_cpus set to {num_cpus}.") + ret = os.system( + f"ray start --head --port {ray_port} --num-cpus {num_cpus} --dashboard-host=0.0.0.0" + ) + if ret != 0: + raise RuntimeError(f"Failed to start ray head with exit code {ret}") + else: + logger.info(f"Successfully started ray head on port {ray_port}.") + + # install an atexit handler to kill the head when we exit + atexit.register(lambda: os.system("ray stop -g 10 --force")) + elif start_workers: + logger.info( + f"Starting ray worker and connecting to {address}. We are process {jax.process_index()}." + ) + logger.info(f"Starting ray worker with num_cpus set to {num_cpus}.") + ret = os.system(f"ray start --address {address} --num-cpus {num_cpus}") + if ret != 0: + raise RuntimeError(f"Failed to start ray head with exit code {ret}") + else: + logger.info(f"Successfully started ray worker and connected to {address}.") logger.info(f"ray.init(address={repr(address)}, namespace={repr(namespace)}, **{repr(kwargs)})") # Ray has retry logic, so we don't need to retry here :fingers-crossed: @@ -318,6 +319,9 @@ def _is_this_machine(host): """ Checks if the given host identifies this machine. """ + if host == "localhost" or host == "0.0.0.0": + return True + try: # Get IP addresses of all interfaces machine_ips = [addr[4][0] for addr in socket.getaddrinfo(socket.gethostname(), None)] @@ -330,3 +334,45 @@ def _is_this_machine(host): # Check if the host IP matches any of the machine IPs return any(host_ip == machine_ip for machine_ip in machine_ips) + + +def _remove_if_possible(path): + try: + os.remove(path) + except OSError: + pass + + +def _touch(file_path): + with open(file_path, "a"): + os.utime(file_path, None) + + +def _is_local_leader(): + import atexit + + import filelock + from jax.experimental.multihost_utils import broadcast_one_to_all + + if jax.process_count() == 1: + return True + + import random + + random_id = random.randint(0, 1000000) + random_id = broadcast_one_to_all(random_id) + + lock = filelock.FileLock(f"/tmp/levanter_local_process_zero_lock.{random_id}") + action_performed_file = f"/tmp/levanter_local_process_zero_action_performed.{random_id}" + + try: + with lock.acquire(timeout=0.1): + if not os.path.exists(action_performed_file): + _touch(action_performed_file) + return True # Action needs to be performed + else: + return False # Action already performed + atexit.register(_remove_if_possible, lock.lock_file) + atexit.register(_remove_if_possible, action_performed_file) + except filelock.Timeout: + return False diff --git a/src/levanter/utils/py_utils.py b/src/levanter/utils/py_utils.py index a172b4498..38ecfc49c 100644 --- a/src/levanter/utils/py_utils.py +++ b/src/levanter/utils/py_utils.py @@ -5,7 +5,7 @@ def logical_cpu_core_count(): """Returns the number of logical CPU cores available to the process.""" - num_cpus = os.getenv("SLURM_CPUS_PER_TASK", None) + num_cpus = os.getenv("SLURM_CPUS_ON_NODE", None) if num_cpus is not None: return int(num_cpus) diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index b8f588df4..c22525fd6 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -161,7 +161,6 @@ def make_state(key): (initial_opt_state, initial_key), step=10, checkpoint_path=tmpdir, - exist_ok=True, ) restored_model, (restored_optstate, rkey), step = load_checkpoint( rep_model, @@ -212,7 +211,7 @@ def loss_fn(model, data): assert_trees_not_close(state, rep_state) with tempfile.TemporaryDirectory() as tmpdir: - save_checkpoint(model, state, step=3, checkpoint_path=tmpdir, exist_ok=True) + save_checkpoint(model, state, step=3, checkpoint_path=tmpdir) restored_model, restored_optstate, step = load_checkpoint( rep_model, rep_state, checkpoint_path=tmpdir, discover_latest=False )