From 829e9a784e60d9fe5bdfffbbdb0a18aa306bebf2 Mon Sep 17 00:00:00 2001 From: Jiahao Li Date: Tue, 23 Apr 2024 21:28:40 +0800 Subject: [PATCH] Support p-tuning v2 for ChatGLM family & fix rope theta for 32k/128k seqlen (#289) --- README.md | 5 +- chatglm.cpp | 201 +++++++++++++++--------- chatglm.h | 136 +++++++++++----- chatglm_cpp/__init__.py | 2 +- chatglm_cpp/convert.py | 81 +++++++--- chatglm_test.cpp | 152 ++++++++++++++---- tests/data/glm3_ptuning_v2_model.data | Bin 0 -> 30708 bytes tests/data/glm_ptuning_v2_model.data | Bin 0 -> 53012 bytes tests/test_convert.py | 216 +++++++++++++++++++------- 9 files changed, 573 insertions(+), 220 deletions(-) create mode 100644 tests/data/glm3_ptuning_v2_model.data create mode 100644 tests/data/glm_ptuning_v2_model.data diff --git a/README.md b/README.md index 67addd9b..4e7ddb82 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ C++ implementation of [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B), [ChatGL Highlights: * Pure C++ implementation based on [ggml](https://github.com/ggerganov/ggml), working in the same way as [llama.cpp](https://github.com/ggerganov/llama.cpp). * Accelerated memory-efficient CPU inference with int4/int8 quantization, optimized KV cache and parallel computing. +* P-Tuning v2 and LoRA finetuned models support. * Streaming generation with typewriter effect. * Python binding, web demo, api servers and more possibilities. @@ -68,7 +69,9 @@ You are free to try any of the below quantization types by specifying `-t * `f16`: half precision floating point weights without quantization. * `f32`: single precision floating point weights without quantization. -For LoRA model, add `-l ` flag to merge your LoRA weights into the base model. +For LoRA models, add `-l ` flag to merge your LoRA weights into the base model. For example, run `python3 chatglm_cpp/convert.py -i THUDM/chatglm3-6b -t q4_0 -o chatglm3-ggml-lora.bin -l shibing624/chatglm3-6b-csc-chinese-lora` to merge public LoRA weights from Hugging Face. + +For P-Tuning v2 models using the [official finetuning script](https://github.com/THUDM/ChatGLM3/tree/main/finetune_demo), additional weights are automatically detected by `convert.py`. If `past_key_values` is on the output weight list, the P-Tuning checkpoint is successfully converted. **Build & Run** diff --git a/chatglm.cpp b/chatglm.cpp index d0e70df5..bd3af4b5 100644 --- a/chatglm.cpp +++ b/chatglm.cpp @@ -171,7 +171,7 @@ void ggml_graph_compute_helper(std::vector &buf, ggml_cgraph // for debugging purpose [[maybe_unused]] static inline ggml_tensor *add_zero(ggml_context *ctx, ggml_tensor *tensor) { - ggml_tensor *zeros = ggml_new_tensor(ctx, tensor->type, tensor->n_dims, tensor->ne); + ggml_tensor *zeros = ggml_new_tensor(ctx, GGML_TYPE_F32, tensor->n_dims, tensor->ne); ggml_set_f32(zeros, 0); tensor_to_device(zeros); ggml_tensor *out = tensor_assign_buffers(ggml_add(ctx, tensor, zeros)); @@ -452,15 +452,12 @@ ggml_tensor *RMSNorm::forward(ModelContext *ctx, ggml_tensor *input) const { static ggml_tensor *apply_activation_inplace(ggml_context *ctx, ggml_tensor *hidden_states, ActivationType hidden_act) { switch (hidden_act) { case ActivationType::GELU: - hidden_states = tensor_assign_buffers(ggml_gelu_inplace(ctx, hidden_states)); - break; + return tensor_assign_buffers(ggml_gelu_inplace(ctx, hidden_states)); case ActivationType::SILU: - hidden_states = tensor_assign_buffers(ggml_silu_inplace(ctx, hidden_states)); - break; + return tensor_assign_buffers(ggml_silu_inplace(ctx, hidden_states)); default: CHATGLM_THROW << "Unknown activation type " << (int)hidden_act; } - return hidden_states; } ggml_tensor *BasicMLP::forward(ModelContext *ctx, ggml_tensor *hidden_states) const { @@ -515,9 +512,9 @@ std::string to_string(ModelType model_type) { } static ggml_tensor *apply_rotary_emb_basic(ModelContext *ctx, ggml_tensor *layer, ggml_tensor *position_ids, int n_ctx, - RopeType rope_type, int dim_scale) { - // tensor a (activation) is of shape [qlen, heads, head_size] - // tensor b (position_ids) is of shape [qlen] + RopeType rope_type, float rope_theta, int dim_scale) { + // tensor a (activation) is of shape [s, #h, d] + // tensor b (position_ids) is of shape [s] ggml_context *gctx = ctx->ctx_b.get(); #ifdef GGML_USE_CUBLAS if (!ggml_is_contiguous(layer)) { @@ -526,14 +523,14 @@ static ggml_tensor *apply_rotary_emb_basic(ModelContext *ctx, ggml_tensor *layer #endif const int head_size = layer->ne[0]; const int rope_dim = head_size / dim_scale; - layer = tensor_assign_buffers( - ggml_rope_inplace(gctx, layer, position_ids, rope_dim, (int)rope_type, n_ctx)); // [qlen, heads, head_size] + layer = tensor_assign_buffers(ggml_rope_custom_inplace(gctx, layer, position_ids, rope_dim, (int)rope_type, n_ctx, + rope_theta, 1.f)); // [s, #h, d] return layer; } static ggml_tensor *apply_rotary_emb_glm(ModelContext *ctx, ggml_tensor *layer, ggml_tensor *position_ids, int n_ctx) { - // tensor a (activation) is of shape [qlen, heads, head_size] - // tensor b (position_ids) is of shape [2 * qlen] + // tensor a (activation) is of shape [s, #h, d] + // tensor b (position_ids) is of shape [2 * s] ggml_context *gctx = ctx->ctx_b.get(); const int head_size = layer->ne[0]; @@ -556,9 +553,9 @@ static ggml_tensor *apply_rotary_emb_glm(ModelContext *ctx, ggml_tensor *layer, #endif a1_rope = tensor_assign_buffers( - ggml_rope_inplace(gctx, a1_rope, b1, rope_dim, (int)RopeType::NEOX, n_ctx)); // [qlen, heads, head_size/2] + ggml_rope_inplace(gctx, a1_rope, b1, rope_dim, (int)RopeType::NEOX, n_ctx)); // [s, #h, d/2] a2_rope = tensor_assign_buffers( - ggml_rope_inplace(gctx, a2_rope, b2, rope_dim, (int)RopeType::NEOX, n_ctx)); // [qlen, heads, head_size/2] + ggml_rope_inplace(gctx, a2_rope, b2, rope_dim, (int)RopeType::NEOX, n_ctx)); // [s, #h, d/2] #ifdef GGML_USE_CUBLAS a1_rope = ggml_cpy(gctx, a1_rope, a1); @@ -570,22 +567,48 @@ static ggml_tensor *apply_rotary_emb_glm(ModelContext *ctx, ggml_tensor *layer, return layer; } +[[maybe_unused]] static ggml_tensor *apply_rotary_emb_glm2(ModelContext *ctx, ggml_tensor *layer, + ggml_tensor *position_ids) { + // layer: [s, #h, d], position_ids: [s] + ggml_context *gctx = ctx->ctx_b.get(); +#ifdef GGML_USE_CUBLAS + if (!ggml_is_contiguous(layer)) { + layer = tensor_assign_buffers(ggml_cont(gctx, layer)); + } +#endif + const int head_size = layer->ne[0]; + const int rope_dim = head_size / 2; + ggml_tensor *roped_layer = + tensor_assign_buffers(ggml_rope(gctx, layer, position_ids, rope_dim, (int)RopeType::GPTJ, 0)); // [s, #h, d] + + ggml_tensor *roped_layer_view = tensor_assign_buffers( + ggml_view_3d(gctx, roped_layer, rope_dim, roped_layer->ne[1], roped_layer->ne[2], roped_layer->nb[1], + roped_layer->nb[2], rope_dim * roped_layer->nb[0])); // [s, #h, d/2] + + ggml_tensor *layer_view = + tensor_assign_buffers(ggml_view_3d(gctx, layer, rope_dim, layer->ne[1], layer->ne[2], layer->nb[1], + layer->nb[2], rope_dim * layer->nb[0])); // [s, #h, d/2] + + ggml_build_forward_expand(&ctx->gf, ggml_cpy(gctx, layer_view, roped_layer_view)); + + return roped_layer; +} + static ggml_tensor *apply_rotary_emb(ModelContext *ctx, ggml_tensor *layer, ggml_tensor *position_ids, int n_ctx, - RopeType rope_type, int dim_scale) { + RopeType rope_type, float rope_theta, int dim_scale) { switch (rope_type) { case RopeType::GPTJ: case RopeType::NEOX: - layer = apply_rotary_emb_basic(ctx, layer, position_ids, n_ctx, rope_type, dim_scale); - break; + return apply_rotary_emb_basic(ctx, layer, position_ids, n_ctx, rope_type, rope_theta, dim_scale); case RopeType::CHATGLM: - layer = apply_rotary_emb_glm(ctx, layer, position_ids, n_ctx); - break; + return apply_rotary_emb_glm(ctx, layer, position_ids, n_ctx); + // case RopeType::CHATGLM2: + // return apply_rotary_emb_glm2(ctx, layer, position_ids); case RopeType::DISABLED: - break; + return layer; default: CHATGLM_THROW << "Unknown rope type " << (int)rope_type; } - return layer; } static inline ggml_tensor *apply_attention_mask_causal(ModelContext *ctx, ggml_tensor *attn_scores, int n_past) { @@ -593,16 +616,18 @@ static inline ggml_tensor *apply_attention_mask_causal(ModelContext *ctx, ggml_t } static ggml_tensor *apply_attention_mask_glm(ModelContext *ctx, ggml_tensor *attn_scores, int n_past) { - // attn_scores is of shape [heads, qlen, klen] + // attn_scores: [#h, s, kvs] + // semantic: attn_scores[:, :-1, -1] = -inf ggml_context *gctx = ctx->ctx_b.get(); + const int kvlen = attn_scores->ne[0]; const int qlen = attn_scores->ne[1]; const int num_attention_heads = attn_scores->ne[2]; ggml_tensor *inf = ggml_new_tensor_3d(gctx, attn_scores->type, 1, qlen - 1, num_attention_heads); ggml_set_f32(inf, -INFINITY); tensor_to_device(inf); // TODO: optimize - ggml_tensor *masked_attn_scores = tensor_assign_buffers( - ggml_view_3d(gctx, attn_scores, 1, qlen - 1, num_attention_heads, qlen * ggml_element_size(attn_scores), - qlen * qlen * ggml_element_size(attn_scores), (qlen - 1) * ggml_element_size(attn_scores))); + ggml_tensor *masked_attn_scores = + tensor_assign_buffers(ggml_view_3d(gctx, attn_scores, 1, qlen - 1, num_attention_heads, attn_scores->nb[1], + attn_scores->nb[2], (kvlen - 1) * attn_scores->nb[0])); ggml_build_forward_expand(&ctx->gf, ggml_cpy(gctx, inf, masked_attn_scores)); return attn_scores; } @@ -629,12 +654,12 @@ ggml_tensor *BasicAttention::forward(ModelContext *ctx, ggml_tensor *hidden_stat const int num_shared_q_heads = num_attention_heads / num_kv_heads; const bool is_gqa = num_shared_q_heads > 1; - ggml_tensor *qkv = query_key_value.forward(ctx, hidden_states); // [qlen, hidden + 2 * kv_hidden] + ggml_tensor *qkv = query_key_value.forward(ctx, hidden_states); // [sq, (#h + 2 * #kvh) * d] // split mixed qkv into separate query, key and value - ggml_tensor *query_layer; // [qlen, heads, head_size] - ggml_tensor *key_layer; // [qlen, kv_heads, head_size] - ggml_tensor *value_layer; // [qlen, kv_heads, head_size] + ggml_tensor *query_layer; // [s, #h, d] + ggml_tensor *key_layer; // [s, #kvh, d] + ggml_tensor *value_layer; // [s, #kvh, d] if (interleaved_qkv) { CHATGLM_CHECK(!is_gqa) << "interleaved qkv is not supported for GQA"; @@ -655,42 +680,39 @@ ggml_tensor *BasicAttention::forward(ModelContext *ctx, ggml_tensor *hidden_stat qkv->nb[1], (hidden_size + head_size * num_kv_heads) * ggml_element_size(qkv)); } - query_layer = apply_rotary_emb(ctx, query_layer, position_ids, n_ctx, rope_type, rope_dim_scale); - key_layer = apply_rotary_emb(ctx, key_layer, position_ids, n_ctx, rope_type, rope_dim_scale); + query_layer = apply_rotary_emb(ctx, query_layer, position_ids, n_ctx, rope_type, rope_theta, rope_dim_scale); + key_layer = apply_rotary_emb(ctx, key_layer, position_ids, n_ctx, rope_type, rope_theta, rope_dim_scale); - query_layer = - tensor_assign_buffers(ggml_cont(gctx, ggml_permute(gctx, query_layer, 0, 2, 1, 3))); // [heads, qlen, head_size] + query_layer = tensor_assign_buffers(ggml_cont(gctx, ggml_permute(gctx, query_layer, 0, 2, 1, 3))); // [#h, s, d] if (num_shared_q_heads > 1) { - query_layer = - tensor_assign_buffers(ggml_reshape_3d(gctx, query_layer, head_size, num_shared_q_heads * qlen, - num_kv_heads)); // [kv_heads, shared_qheads * qlen, head_size] + query_layer = tensor_assign_buffers(ggml_reshape_3d(gctx, query_layer, head_size, num_shared_q_heads * qlen, + num_kv_heads)); // [#kvh, (#h/#kvh) * s, d] } - key_layer = tensor_assign_buffers(ggml_permute(gctx, key_layer, 0, 2, 1, 3)); // [kv_heads, qlen, head_size] - - value_layer = tensor_assign_buffers(ggml_permute(gctx, value_layer, 1, 2, 0, 3)); // [kv_heads, head_size, qlen] + key_layer = tensor_assign_buffers(ggml_permute(gctx, key_layer, 0, 2, 1, 3)); // [#kvh, s, d] + value_layer = tensor_assign_buffers(ggml_permute(gctx, value_layer, 1, 2, 0, 3)); // [#kvh, d, s] // store key & value to cache ggml_tensor *k_cache_view = tensor_assign_buffers( ggml_view_3d(gctx, k_cache, head_size, qlen, num_kv_heads, k_cache->nb[1], k_cache->nb[2], - n_past * head_size * ggml_element_size(k_cache))); // [kv_heads, qlen, head_size] + (num_virtual_tokens + n_past) * head_size * ggml_element_size(k_cache))); // [#kvh, s, d] ggml_build_forward_expand(&ctx->gf, ggml_cpy(gctx, key_layer, k_cache_view)); ggml_tensor *v_cache_view = tensor_assign_buffers(ggml_view_3d(gctx, v_cache, qlen, head_size, num_kv_heads, v_cache->nb[1], v_cache->nb[2], - n_past * ggml_element_size(v_cache))); // [kv_heads, head_size, qlen] + (num_virtual_tokens + n_past) * ggml_element_size(v_cache))); // [#kvh, d, s] ggml_build_forward_expand(&ctx->gf, ggml_cpy(gctx, value_layer, v_cache_view)); // concat key & value with past kv - key_layer = tensor_assign_buffers(ggml_view_3d(gctx, k_cache, head_size, n_past + qlen, num_kv_heads, - k_cache->nb[1], k_cache->nb[2], - 0)); // [kv_heads, klen, head_size] - value_layer = tensor_assign_buffers(ggml_view_3d(gctx, v_cache, n_past + qlen, head_size, num_kv_heads, - v_cache->nb[1], v_cache->nb[2], - 0)); // [kv_heads, head_size, klen] + key_layer = tensor_assign_buffers(ggml_view_3d(gctx, k_cache, head_size, num_virtual_tokens + n_past + qlen, + num_kv_heads, k_cache->nb[1], k_cache->nb[2], + 0)); // [#kvh, kvs, d] + value_layer = tensor_assign_buffers(ggml_view_3d(gctx, v_cache, num_virtual_tokens + n_past + qlen, head_size, + num_kv_heads, v_cache->nb[1], v_cache->nb[2], + 0)); // [#kvh, d, kvs] // attention ggml_tensor *attn_scores = - tensor_assign_buffers(ggml_mul_mat(gctx, key_layer, query_layer)); // [kv_heads, shared_qheads * qlen, klen] + tensor_assign_buffers(ggml_mul_mat(gctx, key_layer, query_layer)); // [#kvh, (#h/#kvh) * s, kvs] attn_scores = tensor_assign_buffers(ggml_scale_inplace(gctx, attn_scores, ggml_new_f32(gctx, 1.f / std::sqrt(head_size)))); if (use_alibi) { @@ -699,27 +721,27 @@ ggml_tensor *BasicAttention::forward(ModelContext *ctx, ggml_tensor *hidden_stat if (n_past == 0) { // build attention mask for context input if (num_shared_q_heads > 1) { - attn_scores = ggml_reshape_3d(gctx, attn_scores, n_past + qlen, qlen, - num_attention_heads); // [heads, qlen, klen] + attn_scores = ggml_reshape_3d(gctx, attn_scores, num_virtual_tokens + n_past + qlen, qlen, + num_attention_heads); // [#h, s, kvs] } - attn_scores = apply_attention_mask(ctx, attn_scores, n_past, attn_mask_type); + attn_scores = apply_attention_mask(ctx, attn_scores, num_virtual_tokens + n_past, attn_mask_type); if (num_shared_q_heads > 1) { - attn_scores = ggml_reshape_3d(gctx, attn_scores, n_past + qlen, num_shared_q_heads * qlen, - num_kv_heads); // [kv_heads, shared_qheads * qlen, klen] + attn_scores = + ggml_reshape_3d(gctx, attn_scores, num_virtual_tokens + n_past + qlen, num_shared_q_heads * qlen, + num_kv_heads); // [#kvh, (#h/#kvh) * s, kvs] } } ggml_tensor *attn_probs = - tensor_assign_buffers(ggml_soft_max_inplace(gctx, attn_scores)); // [kv_heads, shared_qheads * qlen, klen] + tensor_assign_buffers(ggml_soft_max_inplace(gctx, attn_scores)); // [#kvh, (#h/#kvh) * s, kvs] - ggml_tensor *context_layer = tensor_assign_buffers( - ggml_mul_mat(gctx, value_layer, attn_probs)); // [kv_heads, shared_qheads * qlen, head_size] + ggml_tensor *context_layer = + tensor_assign_buffers(ggml_mul_mat(gctx, value_layer, attn_probs)); // [#kvh, (#h/#kvh) * s, d] if (num_shared_q_heads > 1) { context_layer = ggml_reshape_3d(gctx, context_layer, head_size, qlen, - num_attention_heads); // [heads, qlen, head_size] + num_attention_heads); // [#h, s, d] } - context_layer = tensor_assign_buffers( - ggml_cont(gctx, ggml_permute(gctx, context_layer, 0, 2, 1, 3))); // [qlen, heads, head_size] - context_layer = tensor_assign_buffers(ggml_reshape_2d(gctx, context_layer, hidden_size, qlen)); // [qlen, hidden] + context_layer = tensor_assign_buffers(ggml_cont(gctx, ggml_permute(gctx, context_layer, 0, 2, 1, 3))); // [s, #h, d] + context_layer = tensor_assign_buffers(ggml_reshape_2d(gctx, context_layer, hidden_size, qlen)); // [s, #h * d] ggml_tensor *attn_output = dense.forward(ctx, context_layer); return attn_output; @@ -730,8 +752,8 @@ BaseModelForCausalLM::BaseModelForCausalLM(ModelConfig config, size_t mem_size, ctx_.dtype = config.dtype; const size_t ctx_w_size = num_weights * ggml_tensor_overhead(); const size_t ctx_kv_size = 2 * config.num_hidden_layers * - (config.max_length * config.hidden_size / config.num_attention_heads * - config.num_kv_heads * ggml_type_size(GGML_TYPE_F16) + + ((config.max_length + config.num_virtual_tokens) * config.hidden_size / + config.num_attention_heads * config.num_kv_heads * ggml_type_size(GGML_TYPE_F16) + ggml_tensor_overhead()); ctx_.ctx_w = make_unique_ggml_context(ctx_w_size, nullptr, true); ctx_.ctx_kv = make_unique_ggml_context(ctx_kv_size + 1 * MB, nullptr, false); // 1MB extra for MPS @@ -1223,6 +1245,21 @@ ChatGLM2ForCausalLM::ChatGLM2ForCausalLM(const ModelConfig &config) } void ChatGLM2ForCausalLM::load(ModelLoader &loader) { + if (config.num_virtual_tokens > 0) { + const int head_size = config.hidden_size / config.num_attention_heads; + auto prefix_cache_ctx = make_unique_ggml_context( + ggml_tensor_overhead() + config.num_hidden_layers * 2 * config.num_kv_heads * config.num_virtual_tokens * + head_size * ggml_type_size(GGML_TYPE_F16), + nullptr, false); + ggml_tensor *past_key_values = + ggml_new_tensor_4d(prefix_cache_ctx.get(), GGML_TYPE_F16, head_size, config.num_virtual_tokens, + config.num_kv_heads, config.num_hidden_layers * 2); + CHATGLM_CHECK(ggml_used_mem(prefix_cache_ctx.get()) == ggml_get_mem_size(prefix_cache_ctx.get())) + << "corrupted prefix cache"; + loader.read_tensor("past_key_values", past_key_values); + load_prefix_cache(past_key_values); + } + std::unordered_map glu_name_map; for (int i = 0; i < config.num_hidden_layers; i++) { std::string layer_prefix = "transformer.encoder.layers." + std::to_string(i) + '.'; @@ -1705,7 +1742,7 @@ Pipeline::Pipeline(const std::string &path, int max_length) { if (max_length > 0) { CHATGLM_CHECK(max_length <= config.max_length) << "Requested max_length (" << max_length << ") exceeds the max possible model sequence length (" - << config.max_length; + << config.max_length << ")"; config.max_length = max_length; } }; @@ -1722,11 +1759,17 @@ Pipeline::Pipeline(const std::string &path, int max_length) { // load version int version = loader.read_basic(); if (model_type == ModelType::CHATGLM) { - CHATGLM_CHECK(version == 1) << "only support version 1 for now but got " << version; - // load config - ModelConfig config(model_type, loader.read_basic(), 1e-5f, ActivationType::GELU, true, true, - true, false, RopeType::CHATGLM, -1, AttentionMaskType::CHATGLM); + ModelConfig config; + if (version == 1) { + config = ModelConfig(model_type, loader.read_basic(), 1e-5f, ActivationType::GELU, true, + true, true, false, RopeType::CHATGLM, 10000.f, -1, AttentionMaskType::CHATGLM, 0); + } else if (version == 2) { + config = ModelConfig(model_type, loader.read_basic(), ActivationType::GELU, true, true, + true, false, RopeType::CHATGLM, -1, AttentionMaskType::CHATGLM); + } else { + CHATGLM_THROW << "only support version 1 or 2 for now but got " << version; + } _update_config_max_length(config, max_length); // load tokenizer @@ -1739,11 +1782,17 @@ Pipeline::Pipeline(const std::string &path, int max_length) { model = std::make_unique(config); model->load(loader); } else if (model_type == ModelType::CHATGLM2 || model_type == ModelType::CHATGLM3) { - CHATGLM_CHECK(version == 1) << "only support version 1 for now but got " << version; - // load config - ModelConfig config(model_type, loader.read_basic(), 1e-5f, ActivationType::SILU, true, false, - false, false, RopeType::GPTJ, 2, AttentionMaskType::CAUSAL); + ModelConfig config; + if (version == 1) { + config = ModelConfig(model_type, loader.read_basic(), 1e-5f, ActivationType::SILU, true, + false, false, false, RopeType::GPTJ, 10000.f, 2, AttentionMaskType::CAUSAL, 0); + } else if (version == 2) { + config = ModelConfig(model_type, loader.read_basic(), ActivationType::SILU, true, false, + false, false, RopeType::GPTJ, 2, AttentionMaskType::CAUSAL); + } else { + CHATGLM_THROW << "only support version 1 or 2 for now but got " << version; + } _update_config_max_length(config, max_length); // load tokenizer @@ -1770,7 +1819,7 @@ Pipeline::Pipeline(const std::string &path, int max_length) { // load config ModelConfig config(model_type, loader.read_basic(), 1e-6f, ActivationType::SILU, false, false, - false, false, RopeType::NEOX, 1, AttentionMaskType::CAUSAL); + false, false, RopeType::NEOX, 10000.f, 1, AttentionMaskType::CAUSAL, 0); _update_config_max_length(config, max_length); // load tokenizer @@ -1789,7 +1838,7 @@ Pipeline::Pipeline(const std::string &path, int max_length) { // load config ModelConfig config(model_type, loader.read_basic(), 1e-6f, ActivationType::SILU, false, false, - false, true, RopeType::DISABLED, -1, AttentionMaskType::CAUSAL); + false, true, RopeType::DISABLED, 10000.f, -1, AttentionMaskType::CAUSAL, 0); _update_config_max_length(config, max_length); // load tokenizer @@ -1811,10 +1860,10 @@ Pipeline::Pipeline(const std::string &path, int max_length) { ModelConfig config; if (rec.hidden_size == 4096) { config = ModelConfig(model_type, rec, 1e-6f, ActivationType::SILU, true, true, false, false, RopeType::NEOX, - 1, AttentionMaskType::CAUSAL); + 10000.f, 1, AttentionMaskType::CAUSAL, 0); } else { config = ModelConfig(model_type, rec, 1e-6f, ActivationType::SILU, false, false, false, false, - RopeType::NEOX, 1, AttentionMaskType::CAUSAL); + RopeType::NEOX, 10000.f, 1, AttentionMaskType::CAUSAL, 0); } _update_config_max_length(config, max_length); diff --git a/chatglm.h b/chatglm.h index 91a6edcd..386ea7e7 100644 --- a/chatglm.h +++ b/chatglm.h @@ -76,10 +76,27 @@ struct ConfigRecordV1 { }; // For compatibility -struct ConfigRecordV2 : public ConfigRecordV1 { +struct ConfigRecordV1GQA : public ConfigRecordV1 { int num_kv_heads; }; +// TODO: use json to serialize config +struct ConfigRecordV2 { + ggml_type dtype; + int vocab_size; + int hidden_size; + int num_attention_heads; + int num_key_value_heads; + int num_hidden_layers; + int intermediate_size; + float norm_eps; + int num_virtual_tokens; + float rope_theta; + int max_length; + int eos_token_id; + int pad_token_id; +}; + enum class ActivationType { GELU, SILU, @@ -89,6 +106,7 @@ enum class RopeType { GPTJ = 0, NEOX = 2, CHATGLM = 4, + CHATGLM2 = 8, DISABLED = 10000, }; @@ -105,33 +123,44 @@ class ModelConfig { ModelConfig(ModelType model_type, ggml_type dtype, int vocab_size, int hidden_size, int num_attention_heads, int num_kv_heads, int num_hidden_layers, int intermediate_size, float norm_eps, ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv, bool use_alibi, - RopeType rope_type, int rope_dim_scale, AttentionMaskType attn_mask_type, int max_length, - int bos_token_id, int eos_token_id, int pad_token_id, int sep_token_id, - std::vector extra_eos_token_ids) + RopeType rope_type, float rope_theta, int rope_dim_scale, AttentionMaskType attn_mask_type, + int num_virtual_tokens, int max_length, int bos_token_id, int eos_token_id, int pad_token_id, + int sep_token_id, std::vector extra_eos_token_ids) : model_type(model_type), dtype(dtype), vocab_size(vocab_size), hidden_size(hidden_size), num_attention_heads(num_attention_heads), num_kv_heads(num_kv_heads), num_hidden_layers(num_hidden_layers), intermediate_size(intermediate_size), norm_eps(norm_eps), hidden_act(hidden_act), use_qkv_bias(use_qkv_bias), use_dense_bias(use_dense_bias), interleaved_qkv(interleaved_qkv), use_alibi(use_alibi), rope_type(rope_type), - rope_dim_scale(rope_dim_scale), attn_mask_type(attn_mask_type), max_length(max_length), - bos_token_id(bos_token_id), eos_token_id(eos_token_id), pad_token_id(pad_token_id), - sep_token_id(sep_token_id), extra_eos_token_ids(std::move(extra_eos_token_ids)) {} + rope_theta(rope_theta), rope_dim_scale(rope_dim_scale), attn_mask_type(attn_mask_type), + num_virtual_tokens(num_virtual_tokens), max_length(max_length), bos_token_id(bos_token_id), + eos_token_id(eos_token_id), pad_token_id(pad_token_id), sep_token_id(sep_token_id), + extra_eos_token_ids(std::move(extra_eos_token_ids)) {} ModelConfig(ModelType model_type, const ConfigRecordV1 &rec, float norm_eps, ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv, bool use_alibi, RopeType rope_type, - int rope_dim_scale, AttentionMaskType attn_mask_type) + float rope_theta, int rope_dim_scale, AttentionMaskType attn_mask_type, int num_virtual_tokens) : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads, rec.num_attention_heads, rec.num_hidden_layers, rec.intermediate_size, norm_eps, hidden_act, - use_qkv_bias, use_dense_bias, interleaved_qkv, use_alibi, rope_type, rope_dim_scale, - attn_mask_type, rec.max_length, rec.bos_token_id, rec.eos_token_id, rec.pad_token_id, - rec.sep_token_id, {}) {} + use_qkv_bias, use_dense_bias, interleaved_qkv, use_alibi, rope_type, rope_theta, rope_dim_scale, + attn_mask_type, num_virtual_tokens, rec.max_length, rec.bos_token_id, rec.eos_token_id, + rec.pad_token_id, rec.sep_token_id, {}) {} - ModelConfig(ModelType model_type, const ConfigRecordV2 &rec, float norm_eps, ActivationType hidden_act, + ModelConfig(ModelType model_type, const ConfigRecordV1GQA &rec, float norm_eps, ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv, bool use_alibi, RopeType rope_type, - int rope_dim_scale, AttentionMaskType attn_mask_type) + float rope_theta, int rope_dim_scale, AttentionMaskType attn_mask_type, int num_virtual_tokens) : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads, rec.num_kv_heads, rec.num_hidden_layers, rec.intermediate_size, norm_eps, hidden_act, use_qkv_bias, use_dense_bias, - interleaved_qkv, use_alibi, rope_type, rope_dim_scale, attn_mask_type, rec.max_length, - rec.bos_token_id, rec.eos_token_id, rec.pad_token_id, rec.sep_token_id, {}) {} + interleaved_qkv, use_alibi, rope_type, rope_theta, rope_dim_scale, attn_mask_type, + num_virtual_tokens, rec.max_length, rec.bos_token_id, rec.eos_token_id, rec.pad_token_id, + rec.sep_token_id, {}) {} + + ModelConfig(ModelType model_type, const ConfigRecordV2 &rec, ActivationType hidden_act, bool use_qkv_bias, + bool use_dense_bias, bool interleaved_qkv, bool use_alibi, RopeType rope_type, int rope_dim_scale, + AttentionMaskType attn_mask_type) + : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads, + rec.num_key_value_heads, rec.num_hidden_layers, rec.intermediate_size, rec.norm_eps, hidden_act, + use_qkv_bias, use_dense_bias, interleaved_qkv, use_alibi, rope_type, rec.rope_theta, + rope_dim_scale, attn_mask_type, rec.num_virtual_tokens, rec.max_length, -1, rec.eos_token_id, + rec.pad_token_id, -1, {}) {} std::string model_type_name() const { return to_string(model_type); } @@ -151,8 +180,10 @@ class ModelConfig { bool interleaved_qkv; bool use_alibi; RopeType rope_type; + float rope_theta; int rope_dim_scale; AttentionMaskType attn_mask_type; + int num_virtual_tokens; int max_length; int bos_token_id; int eos_token_id; @@ -388,16 +419,17 @@ class BasicAttention { BasicAttention() = default; BasicAttention(ModelContext *ctx, int hidden_size, int num_attention_heads, int num_kv_heads, int max_length, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv, bool use_alibi, RopeType rope_type, - int rope_dim_scale, AttentionMaskType attn_mask_type) + float rope_theta, int rope_dim_scale, AttentionMaskType attn_mask_type, int num_virtual_tokens) : num_attention_heads(num_attention_heads), num_kv_heads(num_kv_heads), interleaved_qkv(interleaved_qkv), - use_alibi(use_alibi), rope_type(rope_type), rope_dim_scale(rope_dim_scale), attn_mask_type(attn_mask_type), + use_alibi(use_alibi), rope_type(rope_type), rope_theta(rope_theta), rope_dim_scale(rope_dim_scale), + attn_mask_type(attn_mask_type), num_virtual_tokens(num_virtual_tokens), query_key_value(ctx, hidden_size, hidden_size + 2 * (hidden_size / num_attention_heads) * num_kv_heads, use_qkv_bias), dense(ctx, hidden_size, hidden_size, use_dense_bias), - k_cache(ggml_new_tensor_3d(ctx->ctx_kv.get(), GGML_TYPE_F16, hidden_size / num_attention_heads, max_length, - num_kv_heads)), - v_cache(ggml_new_tensor_3d(ctx->ctx_kv.get(), GGML_TYPE_F16, max_length, hidden_size / num_attention_heads, - num_kv_heads)) {} + k_cache(ggml_new_tensor_3d(ctx->ctx_kv.get(), GGML_TYPE_F16, hidden_size / num_attention_heads, + max_length + num_virtual_tokens, num_kv_heads)), + v_cache(ggml_new_tensor_3d(ctx->ctx_kv.get(), GGML_TYPE_F16, max_length + num_virtual_tokens, + hidden_size / num_attention_heads, num_kv_heads)) {} ggml_tensor *forward(ModelContext *ctx, ggml_tensor *hidden_states, ggml_tensor *position_ids, int n_past, int n_ctx) const; @@ -408,12 +440,14 @@ class BasicAttention { bool interleaved_qkv; bool use_alibi; RopeType rope_type; + float rope_theta; int rope_dim_scale; AttentionMaskType attn_mask_type; + int num_virtual_tokens; Linear query_key_value; Linear dense; - ggml_tensor *k_cache; // [kv_heads, max_len, head_size] - ggml_tensor *v_cache; // [kv_heads, head_size, max_len] + ggml_tensor *k_cache; // [#kvh, s, d] + ggml_tensor *v_cache; // [#kvh, d, s] }; template @@ -422,11 +456,12 @@ class BasicBlock { BasicBlock() = default; BasicBlock(ModelContext *ctx, int hidden_size, int num_attention_heads, int num_kv_heads, int intermediate_size, int max_length, float norm_eps, ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, - bool interleaved_qkv, bool use_alibi, RopeType rope_type, int rope_dim_scale, - AttentionMaskType attn_mask_type) + bool interleaved_qkv, bool use_alibi, RopeType rope_type, float rope_theta, int rope_dim_scale, + AttentionMaskType attn_mask_type, int num_virtual_tokens) : input_layernorm(ctx, hidden_size, false, norm_eps), attention(ctx, hidden_size, num_attention_heads, num_kv_heads, max_length, use_qkv_bias, use_dense_bias, - interleaved_qkv, use_alibi, rope_type, rope_dim_scale, attn_mask_type), + interleaved_qkv, use_alibi, rope_type, rope_theta, rope_dim_scale, attn_mask_type, + num_virtual_tokens), post_attention_layernorm(ctx, hidden_size, false, norm_eps), mlp(ctx, hidden_size, intermediate_size, hidden_act) {} @@ -517,16 +552,44 @@ class BasicModel { return hidden_states; } + void load_prefix_cache(const ModelConfig &config, ggml_tensor *past_key_values) { + ggml_cgraph gf{}; + auto ctx = make_unique_ggml_context(config.num_hidden_layers * 7 * ggml_tensor_overhead(), nullptr, false); + const int head_size = config.hidden_size / config.num_attention_heads; + for (size_t i = 0; i < layers.size(); i++) { + auto &attn = layers[i].attention; + ggml_tensor *virtual_key = ggml_view_3d(ctx.get(), past_key_values, head_size, config.num_virtual_tokens, + config.num_kv_heads, past_key_values->nb[1], past_key_values->nb[2], + i * 2 * past_key_values->nb[3]); // [#h, v, d] + ggml_tensor *k_cache_view = + ggml_view_3d(ctx.get(), attn.k_cache, head_size, config.num_virtual_tokens, config.num_kv_heads, + attn.k_cache->nb[1], attn.k_cache->nb[2], 0); // [#h, v, d] + ggml_build_forward_expand(&gf, ggml_cpy(ctx.get(), virtual_key, k_cache_view)); + + ggml_tensor *virtual_value = ggml_view_3d( + ctx.get(), past_key_values, head_size, config.num_virtual_tokens, config.num_kv_heads, + past_key_values->nb[1], past_key_values->nb[2], (i * 2 + 1) * past_key_values->nb[3]); // [#h, v, d] + virtual_value = ggml_permute(ctx.get(), virtual_value, 1, 0, 2, 3); // [#h, d, v] + ggml_tensor *v_cache_view = + ggml_view_3d(ctx.get(), attn.v_cache, config.num_virtual_tokens, head_size, config.num_kv_heads, + attn.v_cache->nb[1], attn.v_cache->nb[2], 0); // [#h, d, v] + ggml_build_forward_expand(&gf, ggml_cpy(ctx.get(), virtual_value, v_cache_view)); + } + CHATGLM_CHECK(ggml_used_mem(ctx.get()) == ggml_get_mem_size(ctx.get())) << "corrupted prefix cache context"; + std::vector compute_buffer; + ggml_graph_compute_helper(compute_buffer, &gf, 0); + } + private: std::vector build_layers(ModelContext *ctx, const ModelConfig &config) { std::vector layers; layers.reserve(config.num_hidden_layers); for (int layer_id = 0; layer_id < config.num_hidden_layers; layer_id++) { - // TODO: reduce max length? 32k might be too large for cpu inference layers.emplace_back(ctx, config.hidden_size, config.num_attention_heads, config.num_kv_heads, config.intermediate_size, config.max_length, config.norm_eps, config.hidden_act, config.use_qkv_bias, config.use_dense_bias, config.interleaved_qkv, config.use_alibi, - config.rope_type, config.rope_dim_scale, config.attn_mask_type); + config.rope_type, config.rope_theta, config.rope_dim_scale, config.attn_mask_type, + config.num_virtual_tokens); } return layers; } @@ -745,6 +808,8 @@ class BasicModelForCausalLM : public BaseModelForCausalLM { return lm_logits; } + void load_prefix_cache(ggml_tensor *past_key_values) { transformer.load_prefix_cache(config, past_key_values); } + protected: void to_cpu() { for (auto &item : state_dict_) { @@ -818,13 +883,14 @@ class GLMBlock : public BasicBlock { GLMBlock() = default; GLMBlock(ModelContext *ctx, int hidden_size, int num_attention_heads, int num_kv_heads, int intermediate_size, int max_length, float norm_eps, ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, - bool interleaved_qkv, bool use_alibi, RopeType rope_type, int rope_dim_scale, - AttentionMaskType attn_mask_type) - : BasicBlock( - LayerNorm(ctx, hidden_size, false, norm_eps), - BasicAttention(ctx, hidden_size, num_attention_heads, num_attention_heads, max_length, use_qkv_bias, - use_dense_bias, interleaved_qkv, use_alibi, rope_type, rope_dim_scale, attn_mask_type), - LayerNorm(ctx, hidden_size, false, norm_eps), BasicMLP(ctx, hidden_size, intermediate_size, hidden_act)), + bool interleaved_qkv, bool use_alibi, RopeType rope_type, float rope_theta, int rope_dim_scale, + AttentionMaskType attn_mask_type, int num_virtual_tokens) + : BasicBlock(LayerNorm(ctx, hidden_size, false, norm_eps), + BasicAttention(ctx, hidden_size, num_attention_heads, num_attention_heads, max_length, + use_qkv_bias, use_dense_bias, interleaved_qkv, use_alibi, rope_type, rope_theta, + rope_dim_scale, attn_mask_type, num_virtual_tokens), + LayerNorm(ctx, hidden_size, false, norm_eps), + BasicMLP(ctx, hidden_size, intermediate_size, hidden_act)), alpha_value(std::sqrt(2.f * 28)) {} ggml_tensor *forward(ModelContext *ctx, ggml_tensor *hidden_states, ggml_tensor *position_ids, int n_past, diff --git a/chatglm_cpp/__init__.py b/chatglm_cpp/__init__.py index 92d9b724..babfac5c 100644 --- a/chatglm_cpp/__init__.py +++ b/chatglm_cpp/__init__.py @@ -6,7 +6,7 @@ import chatglm_cpp._C as _C from chatglm_cpp._C import ChatMessage -__version__ = "0.3.1" +__version__ = "0.3.2" @dataclass diff --git a/chatglm_cpp/convert.py b/chatglm_cpp/convert.py index 87b6924d..a98168c1 100644 --- a/chatglm_cpp/convert.py +++ b/chatglm_cpp/convert.py @@ -128,8 +128,6 @@ def quantize_q5_1(tensor: torch.Tensor) -> torch.Tensor: def dump_tensor(f, name: str, tensor: torch.Tensor, ggml_type: GGMLType): - assert tensor.dtype == torch.float32 - # tensor name f.write(struct.pack("i", len(name.encode()))) f.write(name.encode()) @@ -165,7 +163,9 @@ def dump_state_dict(f, weight_names, state_dict, quantization_bit, ggml_type): tensor_info = [] for name in tqdm(weight_names, desc="Processing model states"): tensor = state_dict[name] - if tensor.ndim == 2: + if name == "past_key_values": + tensor_ggml_type = GGMLType.F16 + elif tensor.ndim == 2: # 2d weight: should quantize it if needed # step 1: de-quantize it back to float32 @@ -191,7 +191,7 @@ def dump_state_dict(f, weight_names, state_dict, quantization_bit, ggml_type): tensor_ggml_type = GGMLType.F32 dump_tensor(f, name, tensor, tensor_ggml_type) - tensor_info.append((name, tensor.shape, tensor_ggml_type.name)) + tensor_info.append((name, tuple(tensor.shape), tensor_ggml_type.name)) print(tabulate(tensor_info, headers=["name", "shape", "dtype"], tablefmt="psql")) @@ -200,12 +200,25 @@ class BaseConverter: @classmethod def convert(cls, f, model, tokenizer, ggml_type): f.write(b"ggml") # magic - f.write(struct.pack("ii", cls.MODEL_TYPE.value, 1)) # model type & version + f.write(struct.pack("i", cls.MODEL_TYPE.value)) # model type cls.dump_config(f, model.config, ggml_type) cls.dump_tokenizer(f, tokenizer) cls.dump_model(f, model, ggml_type) +def get_prefix_cache(prefix_encoder, pre_seq_len, num_layers, num_kv_heads, head_size): + prefix_tokens = torch.arange(pre_seq_len, dtype=torch.long) + with torch.no_grad(): + past_key_values = prefix_encoder(prefix_tokens) + past_key_values = ( + past_key_values.to(torch.half) + .view(pre_seq_len, num_layers * 2, num_kv_heads, head_size) + .permute(1, 2, 0, 3) + .contiguous() + ) + return past_key_values + + class ChatGLMConverter(BaseConverter): MODEL_TYPE = ModelType.CHATGLM @@ -215,20 +228,24 @@ def dump_config(f, config, ggml_type): assert ( config.inner_hidden_size == 4 * config.hidden_size ), "unimplemented: inner_hidden_size should be 4 times hidden_size" + + config_version = 2 config_values = [ ggml_type.value, config.vocab_size, config.hidden_size, config.num_attention_heads, + config.num_attention_heads, config.num_layers, config.inner_hidden_size, + config.layernorm_epsilon, + config.pre_seq_len if config.pre_seq_len is not None else 0, + 10000.0, # rope_theta config.max_sequence_length, - config.bos_token_id if config.bos_token_id is not None else -1, config.eos_token_id if config.eos_token_id is not None else -1, config.pad_token_id if config.pad_token_id is not None else -1, - config.sep_token_id if config.sep_token_id is not None else -1, ] - f.write(struct.pack("i" * len(config_values), *config_values)) + f.write(struct.pack("iiiiiiiififiii", config_version, *config_values)) @staticmethod def dump_tokenizer(f, tokenizer): @@ -268,8 +285,8 @@ def dump_model(f, model, ggml_type): class ChatGLM2Converter(BaseConverter): MODEL_TYPE = ModelType.CHATGLM2 - @staticmethod - def dump_config(f, config, ggml_type): + @classmethod + def dump_config(cls, f, config, ggml_type): assert config.add_bias_linear is False, "unimplemented: add_bias_linear must be false" assert config.add_qkv_bias is True, "unimplemented: add_qkv_bias must be true" assert ( @@ -283,22 +300,24 @@ def dump_config(f, config, ggml_type): assert config.post_layer_norm is True, "unimplemented: post_layer_norm must be true" assert config.rmsnorm is True, "unimplemented: rmsnorm must be true" + config_version = 2 config_values = [ ggml_type.value, config.padded_vocab_size, config.hidden_size, config.num_attention_heads, + config.multi_query_group_num, config.num_layers, config.ffn_hidden_size, + config.layernorm_epsilon, + config.pre_seq_len if config.pre_seq_len is not None else 0, + 10000.0 * getattr(config, "rope_ratio", 1), # rope_theta config.seq_length, - config.bos_token_id if config.bos_token_id is not None else -1, config.eos_token_id if config.eos_token_id is not None else -1, config.pad_token_id if config.pad_token_id is not None else -1, - config.sep_token_id if config.sep_token_id is not None else -1, - config.multi_query_group_num, ] - f.write(struct.pack("i" * len(config_values), *config_values)) + f.write(struct.pack("iiiiiiiififiii", config_version, *config_values)) @staticmethod def dump_tokenizer(f, tokenizer): @@ -308,8 +327,24 @@ def dump_tokenizer(f, tokenizer): @staticmethod def dump_model(f, model, ggml_type): - weight_names = ["transformer.embedding.word_embeddings.weight"] - for i in range(model.config.num_layers): + config = model.config + + state_dict = model.state_dict() + + weight_names = [] + if config.pre_seq_len is not None and config.pre_seq_len > 0: + past_key_values = get_prefix_cache( + model.transformer.prefix_encoder, + config.pre_seq_len, + config.num_layers, + config.multi_query_group_num, + config.kv_channels, + ) + state_dict["past_key_values"] = past_key_values + weight_names.append("past_key_values") + + weight_names.append("transformer.embedding.word_embeddings.weight") + for i in range(config.num_layers): weight_names += [ f"transformer.encoder.layers.{i}.input_layernorm.weight", f"transformer.encoder.layers.{i}.self_attention.query_key_value.weight", @@ -323,7 +358,7 @@ def dump_model(f, model, ggml_type): "transformer.encoder.final_layernorm.weight", "transformer.output_layer.weight", ] - dump_state_dict(f, weight_names, model.state_dict(), model.config.quantization_bit, ggml_type) + dump_state_dict(f, weight_names, state_dict, config.quantization_bit, ggml_type) class ChatGLM3Converter(ChatGLM2Converter): @@ -331,10 +366,11 @@ class ChatGLM3Converter(ChatGLM2Converter): class BaichuanConverter(BaseConverter): - @staticmethod - def dump_config(f, config, ggml_type): + @classmethod + def dump_config(cls, f, config, ggml_type): assert config.hidden_act == "silu", "unimplemented: hidden_act must be silu" + config_version = 1 config_values = [ ggml_type.value, config.vocab_size, @@ -349,7 +385,7 @@ def dump_config(f, config, ggml_type): config.sep_token_id if config.sep_token_id is not None else -1, ] - f.write(struct.pack("i" * len(config_values), *config_values)) + f.write(struct.pack("i" * (1 + len(config_values)), config_version, *config_values)) @staticmethod def dump_tokenizer(f, tokenizer): @@ -397,6 +433,7 @@ class InternLMConverter(BaseConverter): def dump_config(f, config, ggml_type): assert config.hidden_act == "silu", "unimplemented: hidden_act must be silu" + config_version = 1 config_values = [ ggml_type.value, config.vocab_size, @@ -411,7 +448,7 @@ def dump_config(f, config, ggml_type): config.sep_token_id if config.sep_token_id is not None else -1, ] - f.write(struct.pack("i" * len(config_values), *config_values)) + f.write(struct.pack("i" * (1 + len(config_values)), config_version, *config_values)) @staticmethod def dump_tokenizer(f, tokenizer): @@ -485,6 +522,8 @@ def convert(f: BinaryIO, model_name_or_path: str, lora_model_name_or_path: Optio model = PeftModel.from_pretrained(model, lora_model_name_or_path) model = model.merge_and_unload() + model = model.eval() + if model.config.model_type == "chatglm": if hasattr(model.config, "multi_query_attention"): # ChatGLM3 shares the same architecture and model config with ChatGLM2, but its tokenizer further supports system prompts, diff --git a/chatglm_test.cpp b/chatglm_test.cpp index 3daf5351..fcfe53ac 100644 --- a/chatglm_test.cpp +++ b/chatglm_test.cpp @@ -22,11 +22,22 @@ static inline void expect_all_close(ggml_tensor *a, ggml_tensor *b, float atol = ASSERT_EQ(a->type, GGML_TYPE_F32); ASSERT_EQ(ggml_nelements(a), ggml_nelements(b)); int64_t numel = ggml_nelements(a); + float max_abs_diff = 0.f; + int64_t num_mismatch = 0; for (int64_t i = 0; i < numel; i++) { float ai = ((float *)a->data)[i]; float bi = ((float *)b->data)[i]; - EXPECT_LT(std::abs(ai - bi), atol + rtol * std::abs(bi)) << "diff " << ai << " vs " << bi; + float abs_diff = std::abs(ai - bi); + max_abs_diff = std::max(max_abs_diff, abs_diff); + if (abs_diff >= atol + rtol * std::abs(bi)) { + num_mismatch++; + } } + EXPECT_EQ(num_mismatch, 0) << "Tensors are not close!\n\n" + << "Mismatched elements: " << num_mismatch << " / " << numel << " (" + << num_mismatch * 100 / numel << "%)\n" + << "Greatest absolute difference: " << max_abs_diff << " (up to " << std::scientific + << atol << " allowed)\n"; } static inline char *read_tensor_data(char *ptr, ggml_tensor *tensor) { @@ -277,16 +288,13 @@ class ChatGLMTest : public ::testing::Test { float perf_device_graph_compute() { return _perf_graph_compute_impl(); } template - void test_model(const Model &model, const ModelConfig &config, const fs::path &data_path, int seq_len, + void test_model(Model &model, const ModelConfig &config, const fs::path &data_path, int seq_len, const std::vector &all_weights) { ASSERT_EQ(config.num_hidden_layers, 1); MappedFile mapped_file(data_path.string()); char *ptr = mapped_file.data; - tensor_to_device(model.layers[0].attention.k_cache); - tensor_to_device(model.layers[0].attention.v_cache); - ggml_tensor *x1 = ggml_new_tensor_1d(ctx.ctx_b.get(), GGML_TYPE_I32, seq_len); ggml_tensor *ref_y1 = ggml_new_tensor_2d(ctx.ctx_b.get(), GGML_TYPE_F32, config.hidden_size, seq_len); ggml_tensor *x2 = ggml_new_tensor_1d(ctx.ctx_b.get(), GGML_TYPE_I32, 1); @@ -299,6 +307,18 @@ class ChatGLMTest : public ::testing::Test { std::vector cpu_tensors{model.word_embeddings.weight, x1, x2, x3}; + if (config.num_virtual_tokens > 0) { + const int head_size = config.hidden_size / config.num_attention_heads; + ggml_tensor *past_key_values = + ggml_new_tensor_4d(ctx.ctx_b.get(), GGML_TYPE_F16, head_size, config.num_virtual_tokens, + config.num_kv_heads, config.num_hidden_layers * 2); // [l * 2, #h, v, d] + ptr = read_tensor_data(ptr, past_key_values); + model.load_prefix_cache(config, past_key_values); + } + + tensor_to_device(model.layers[0].attention.k_cache); + tensor_to_device(model.layers[0].attention.v_cache); + for (auto tensor : all_tensors) { ptr = read_tensor_data(ptr, tensor); if (std::find(cpu_tensors.begin(), cpu_tensors.end(), tensor) == cpu_tensors.end()) { @@ -310,6 +330,7 @@ class ChatGLMTest : public ::testing::Test { // self attention { + reset_cgraph(); ggml_tensor *out_y1 = model.forward(&ctx, x1, 0, seq_len); EXPECT_EQ(out_y1->backend, ref_y1->backend); out_y1->backend = GGML_BACKEND_CPU; @@ -320,8 +341,8 @@ class ChatGLMTest : public ::testing::Test { } // cross attention - reset_cgraph(); { + reset_cgraph(); ggml_tensor *out_y2 = model.forward(&ctx, x2, seq_len, seq_len); EXPECT_EQ(out_y2->backend, ref_y2->backend); out_y2->backend = GGML_BACKEND_CPU; @@ -330,8 +351,8 @@ class ChatGLMTest : public ::testing::Test { expect_all_close(ref_y2, out_y2, 5e-4); } - reset_cgraph(); { + reset_cgraph(); ggml_tensor *out_y3 = model.forward(&ctx, x3, seq_len + 1, seq_len); EXPECT_EQ(out_y3->backend, ref_y3->backend); out_y3->backend = GGML_BACKEND_CPU; @@ -585,8 +606,45 @@ TEST_F(ChatGLMTest, GLMModel) { ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/128, /*norm_eps=*/1e-5f, /*hidden_act=*/ActivationType::GELU, /*use_qkv_bias=*/true, /*use_dense_bias=*/true, - /*interleaved_qkv=*/true, /*use_alibi=*/false, /*rope_type=*/RopeType::CHATGLM, /*rope_dim_scale=*/-1, - /*attn_mask_type=*/AttentionMaskType::CHATGLM, + /*interleaved_qkv=*/true, /*use_alibi=*/false, /*rope_type=*/RopeType::CHATGLM, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/-1, + /*attn_mask_type=*/AttentionMaskType::CHATGLM, /*num_virtual_tokens=*/0, + /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, + /*extra_eos_token_ids=*/{}); + + constexpr int seq_len = 3; + + ChatGLMModel model(&ctx, config); + + std::vector all_weights{model.word_embeddings.weight, + model.layers[0].input_layernorm.weight, + model.layers[0].input_layernorm.bias, + model.layers[0].attention.query_key_value.weight, + model.layers[0].attention.query_key_value.bias, + model.layers[0].attention.dense.weight, + model.layers[0].attention.dense.bias, + model.layers[0].post_attention_layernorm.weight, + model.layers[0].post_attention_layernorm.bias, + model.layers[0].mlp.dense_h_to_4h.weight, + model.layers[0].mlp.dense_h_to_4h.bias, + model.layers[0].mlp.dense_4h_to_h.weight, + model.layers[0].mlp.dense_4h_to_h.bias, + model.final_layernorm.weight, + model.final_layernorm.bias}; + + test_model(model, config, data_path, seq_len, all_weights); +} + +TEST_F(ChatGLMTest, GLMPTuningV2Model) { + fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm_ptuning_v2_model.data"; + + ModelConfig config( + ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, + /*num_kv_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/128, /*norm_eps=*/1e-5f, + /*hidden_act=*/ActivationType::GELU, /*use_qkv_bias=*/true, /*use_dense_bias=*/true, + /*interleaved_qkv=*/true, /*use_alibi=*/false, /*rope_type=*/RopeType::CHATGLM, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/-1, + /*attn_mask_type=*/AttentionMaskType::CHATGLM, /*num_virtual_tokens=*/5, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -620,8 +678,9 @@ TEST_F(ChatGLMTest, GLM2Model) { ModelType::CHATGLM2, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/true, /*use_dense_bias=*/false, - /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::GPTJ, /*rope_dim_scale=*/2, - /*attn_mask_type=*/AttentionMaskType::CAUSAL, + /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::GPTJ, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/2, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/0, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -629,9 +688,6 @@ TEST_F(ChatGLMTest, GLM2Model) { ChatGLM2Model model(&ctx, config); - tensor_to_device(model.layers[0].attention.k_cache); - tensor_to_device(model.layers[0].attention.v_cache); - std::vector all_weights{model.word_embeddings.weight, model.layers[0].input_layernorm.weight, model.layers[0].attention.query_key_value.weight, @@ -653,8 +709,9 @@ TEST_F(ChatGLMTest, GLM3Model) { ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/true, /*use_dense_bias=*/false, - /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::GPTJ, /*rope_dim_scale=*/2, - /*attn_mask_type=*/AttentionMaskType::CAUSAL, + /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::GPTJ, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/2, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/0, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -662,8 +719,36 @@ TEST_F(ChatGLMTest, GLM3Model) { ChatGLM3Model model(&ctx, config); - tensor_to_device(model.layers[0].attention.k_cache); - tensor_to_device(model.layers[0].attention.v_cache); + std::vector all_weights{model.word_embeddings.weight, + model.layers[0].input_layernorm.weight, + model.layers[0].attention.query_key_value.weight, + model.layers[0].attention.query_key_value.bias, + model.layers[0].attention.dense.weight, + model.layers[0].post_attention_layernorm.weight, + model.layers[0].mlp.gate_proj.weight, + model.layers[0].mlp.up_proj.weight, + model.layers[0].mlp.down_proj.weight, + model.final_layernorm.weight}; + + test_model(model, config, data_path, seq_len, all_weights); +} + +TEST_F(ChatGLMTest, GLM3PTuningV2Model) { + fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm3_ptuning_v2_model.data"; + + ModelConfig config( + ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, + /*num_kv_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f, + /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/true, /*use_dense_bias=*/false, + /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::GPTJ, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/2, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/5, + /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, + /*extra_eos_token_ids=*/{}); + + constexpr int seq_len = 3; + + ChatGLM3Model model(&ctx, config); std::vector all_weights{model.word_embeddings.weight, model.layers[0].input_layernorm.weight, @@ -686,8 +771,9 @@ TEST_F(ChatGLMTest, Baichuan7BModel) { ModelType::BAICHUAN7B, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/32 * 3, /*norm_eps=*/1e-6f, /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/false, /*use_dense_bias=*/false, - /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::NEOX, /*rope_dim_scale=*/1, - /*attn_mask_type=*/AttentionMaskType::CAUSAL, + /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::NEOX, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/1, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/0, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -715,8 +801,9 @@ TEST_F(ChatGLMTest, Baichuan13BModel) { ModelType::BAICHUAN13B, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/32 * 3, /*norm_eps=*/1e-6f, /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/false, /*use_dense_bias=*/false, - /*interleaved_qkv=*/false, /*use_alibi=*/true, /*rope_type=*/RopeType::DISABLED, /*rope_dim_scale=*/-1, - /*attn_mask_type=*/AttentionMaskType::CAUSAL, + /*interleaved_qkv=*/false, /*use_alibi=*/true, /*rope_type=*/RopeType::DISABLED, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/-1, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/0, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -744,8 +831,9 @@ TEST_F(ChatGLMTest, InternLMModel) { ModelType::INTERNLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8, /*num_kv_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/32 * 3, /*norm_eps=*/1e-6f, /*hidden_act=*/ActivationType::SILU, /*use_qkv_bias=*/true, /*use_dense_bias=*/true, - /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::NEOX, /*rope_dim_scale=*/1, - /*attn_mask_type=*/AttentionMaskType::CAUSAL, + /*interleaved_qkv=*/false, /*use_alibi=*/false, /*rope_type=*/RopeType::NEOX, /*rope_theta=*/10000.f, + /*rope_dim_scale=*/1, + /*attn_mask_type=*/AttentionMaskType::CAUSAL, /*num_virtual_tokens=*/0, /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1, /*extra_eos_token_ids=*/{}); @@ -1128,7 +1216,8 @@ TEST(Pipeline, ChatGLM3) { { ChatMessage output = pipeline.chat(messages, gen_config); EXPECT_EQ(output.role, ChatMessage::ROLE_ASSISTANT); - EXPECT_EQ(output.content, "根据您的要求，我使用随机数生成器API生成了一个在0和100之间的随机数，结果为22。"); + EXPECT_EQ(output.content, + "根据API调用结果，我为您生成了一个随机数，随机数的范围在0到100之间。这个随机数是22。"); } } @@ -1143,11 +1232,14 @@ TEST(Pipeline, ChatGLM3) { { ChatMessage output = pipeline.chat(messages, gen_config); EXPECT_EQ(output.role, ChatMessage::ROLE_ASSISTANT); - EXPECT_EQ(output.content, "好的，我会为您列出100以内的所有质数。\n\n质数是指只能被1和它本身整除的大于1" - "的整数。例如，2、3、5、7等都是质数。\n\n让我们开始吧！"); + EXPECT_EQ(output.content, R"(好的，我会为您列出100以内的所有质数。 + +质数是指只能被1和它本身整除的正整数。例如，2、3、5、7等都是质数。 + +让我们开始吧！)"); EXPECT_EQ(output.tool_calls.front().code.input, R"(```python +# Function to check if a number is prime def is_prime(n): - """Check if a number is prime.""" if n <= 1: return False if n <= 3: @@ -1162,8 +1254,8 @@ def is_prime(n): return True # Get all prime numbers up to 100 -primes_upto_100 = [i for i in range(2, 101) if is_prime(i)] -primes_upto_100 +primes_up_to_100 = [i for i in range(2, 101) if is_prime(i)] +primes_up_to_100 ```)"); messages.emplace_back(std::move(output)); } diff --git a/tests/data/glm3_ptuning_v2_model.data b/tests/data/glm3_ptuning_v2_model.data new file mode 100644 index 0000000000000000000000000000000000000000..730a0a49f95f8ef843458665f5bcff2268933e50 GIT binary patch literal 30708 zcmWife^ku*_s55AX^BljVlxOK86<7y^)yIAXbFi;Y;1^6OKeNalw?X6BuO%oBuSEf z&g&XUl4K+qNk)<+N!o&lxzP=hEDLX z*_q{d?eXO<8Mh|PwN0`6JYlO{)TFAB%O@|n7L`?KH)!0`)Rrq_rcAqd?&`y9)$%rK ze%QhMUK)|8-#unn{+!nj@8E;o=Tl?nfAA62lDo8%J4W;*-^sRAe>VsRPrXFd(1$P@ z@eeirxr!WT%!B2IFjSqbAoVaG3S9OVc`cjGrC%pe)5}{Vd3%xT^BTF_?iUt2awge# zZl>bHwWtm6OZhvZQT^9>GH(5y)XQ99KfaiWd)}a7^|#=&rYDKyeRH(F`?-I}O=Q=6 zVpRt>qdB*l7qjp1oi-cBd4XuYSja~+IzOh)dD+oYIrfp0ldMUo2#Nq-`nDRsRd?dC+v>9&*&FrJV-@L69Hq1|2~2kJ8p$krsl`7{qcJAmkk)2KE;bWU6jQ)FFSJ2k zmOM4h?=U=aF42*cY;xNh0&}C&NJ136R_BBh4~ z1Gc8f{JSLt9`wdKr%7RiTUrLH@BmFLTrds=ZK85r2&(NAoq*`_DQ^mVV-{ zp?`C|I)|l7BK%whSWZGFZ0?xk~{ofh~Q086#u*pqUe)TwfC>WdH4c^ z@7GYhDhrNm5`{`q$jZbE(Fr%0UA~-34*JoS2hXvi`8$dKI>I#a5z<`m5*mES6Q&KX zNIuSA?e&X}XSW@Krq?sx{J4>tYX(Pw%I_c zdK=EXCR;IyjX8^*R80;~j*`l;5e@QacujgF)wF(qPqia1Werz<@yjL#DQdTZ>g1gD|h<$7zG5ivl3|)O0Gn~wXrwVE;|B!?EoG> zy9FK#zESIopRmBBhT1=GMfmG1w9PDlbL%K_-Db-ahTRaT7*8%0?@>GdGCDT>i%5kl zm36%eJD*^R`$tR0?qbsHlk&u%GbpiMjF1R>q)Z${hB<{;x1tpbg!l7hX&s_(7qP)X z6QS?#j1KE#5FPnC)%NiZ6cQ+*(1i))m2^ry?Ajp;xfP9r&RH}utQz&kHB>+D5wu_b zWW9g;K&^WRqD`AXxy8k#Ir&&BN$EqXYaPsFTQTBpH&ET_PI$H+Ael!ocldA)GHhB!mC`ii>M?hR6U{|-9whK%N9lYZ4uCU1;YCmkO|-m9(P`+O;?k4KWB za|sV|oq~|CU!l=V;prK2xPG!jk6F#sa=ev9W?@o^<9Jx?%A_ri_E3_lm9krIFqeoN z7-(w{?xz*r*DWae%%_m;FUWgt0&IVI3^~fw*?szwU-fQQFx3zKg}#Vrk_$CLA&qV_ zLiVyqCTS>V#$DgA=A7|;aYrzRt_zMEU%K%Y~GQ7;6()dzK04!@+judM2b9q zl$6tINVR(pTN(6}B;9|eV&^lM=zWVqj8jp!?i<>hPIABe?Y!9MBUMJdL&~6g@H()9 z*KCeP>)$UZ%J~xIMl?e+zLS(cx%09R1!bmwAXnQYRyA}L85ZARqDj-p$Z!mb-1f21 z1)Zd|K8xsCaoEv$lN2Y-5E;0I*64)0Ib}N;N2y@DDxUB>j8fLRA~2{YBK})XUQ#vm z=ki%|ol)Ut^+l6LtHBXl^Jj zCDHYRYWYJyspXqkB%I#{UB63g!Qc$YqgHUU_xI5Bv7SUBe(J!vYpHX30;B_fCiJ^R zeyJ&3q>1{~VqOZWema2k(AQM&;Z8=214w`Gv0A+A4=6e!d2r7+@Jm|FtkMraGp3Cz zeIlWKB-~f!ajE#QLY<&n1LIgF)BJgcl5`ei`0hNpDqETQXAPR$kC0p1SeTr>MYZ*d zkl3@CTFvK>{8V0^Rn{e*(MJjGXcrP)8x1497h0QoBH6T^WFHTZ#^x&*eKl0u$2vkH zoX5=LA=50mOq5QXX;B7$C$@{SQ1xjRB!C|7H8lAv95nR^|Tv+Sn#=?Q68uxjTibGv8DDnw?Oz6mfCwuTYl%$+X=@q0aa*$&O5>l~asJ`#y&{>X$>Y z@Gxp^ex>c7QfcDMy`)QSq8(=vsl%`rIoBnz=-h3z>&gRi{VfRk_!4NW^QFr8!#w4m zH6;0OB)md~Gx@*`A*Od52rk5i>r12hAMsWtMqWa1Q$@Wce6cluHL z?%S+e&NVm>NMOM)D=B1QDv9eDlq-y=Yb6zVG_dl<; zmRmuTeLcsxTPHVo<174pCHl`(AmoS@iK3rR)xODR8}3I^*P&0)Vzm$A=f7j(fOn8- zLb-I}4K$DN6U+?H6ueHULb+|SqmvVJz;#el1KmUilm+On0uxanwlfh zR;xgy%zsCs#RHi8^${r32g6lwkENF{!cW(m8C)C*<0)Us<)3X7P_!PBFA|s@_kg}? z0V)>g$n=lV&@3ECNu}qwIB6)iG0?$v&MUNk{};Ytvq}AE05ld7Ea+bV`MRMjW6lRk z$(#?PqPryjFNBGvoQLI_#b}?^0*m`?h_-AXQQjPBxwbEieTsRzQ3%-|XoE$<3Cg|Q zNV=DAsCDgrQm$FXa61Kh@9%1_pYD+2HHY}G0legnJ!uAPQ9G6i=l#5!8=RN}@1K?; z{%R7XOss=j!Uf3Q<*Vc0C!lprFY$+H`mpzswZ*ZNO9U`aA*;>U9rhqIaRqW0-mNEY6=Z|^#L{SFCsnTE=f+7QPrc{ zBwIa1+H6qGzlgt(*~UI3TB(B4-Gt}PE1}YHN#r_W8vK71qr={lYE=&?Enqopa~0Ii z^c2Ax;8-~yW;vI5{ro8A*K{A9@|Psfy35;LCX@HYb}Bc2g19dqNcDX_g|`KvCG|B@ ze(g@iM&%GK7^S9TjkNuA6Upl~k*KG0o+$o4+^%|%PevR!>UtT>K62(2FoNsqZ}HZDmr>Gt33LnkQ9?Pe;OQR7gC;}% z*^i34rBTt-o)nlh9OBHCED&F?qbUWx;s2w;H(A*6;V_jvuqMmv9G>IUWHhfgsn?c~ z;>0E-ZplJz&lhM8m`EGuaT+sVAHEzsN7A@4q&{6niC1pJt9U+_*(sQqy_foJKEnKe z^P=cSU*Kt3PiY-{`6ArK-iXRQ}!&3*25q z{@*F)@{XA;00{nWObk_}-6!X5|YmQr(jFPi=1*#{Wx^t=8b6G5Z`Q(nS`0N$btJDTRZ_ovD<<-|FZG)c#`SY8N=>=x3@?jZBs zRgjMJB>N9vm?+8tepxD}y?TaZ{XALn$@5g!@)+9waV+yqHc}sFz~{qqSU$EzzNZo{ zlh4wku{WR)S8+VK1^F*O@xYmzVBCX4aUh@R!wb3NBUd;M{7%iI7IDcAEiddI3z3q*dNMauUJ zSz+USlou9J(b5Y{WcP>EiiJ>8YAKu-rXfwzjmw=@a;g7mXxEM=pC%SMmaEZL0|pN&Yvh?2t}V~048wtT2X|J7iysz zIW75!dKZSJJ@?bdh#TZKbu_i#JHZ0N*HK-5BmzBmlI0;Y*!`A`Me=JDtiMe*x>kz5 zqG8)RijckNEXfZ)1`kyUC3F5!|arO0L zSYG=^A&-X9qNV>)`#;SrX>J3r_@j(EObf`ThY5;))u5_xB}~#Tqcf$H{O*?{v|B60 z8w#bJxBesXfE+5gR71ue5?KDeGI&l&fUL_u!oByTqERiZbxjzhJ)g;wEKN}#IF%Cq zngVU{VcsM1Efh=2NZL<_?FlJpp7x5`&o#4vfqQt>UO%K5_|p~&Mo72WLQuF2M1f431Csm7!q@)NI{co+vg_QJ>X zJjHlcQ^tzvRPx7gsAE4OplTgD#F6lg`~?NWyON?aiQBjdcmJ14)*Ms967L^_aacBN z%}t=Xw1E3;TSFZOPLlh_-L%eK3x)PBGM4Y7@c$mebhZH*tnNbPp~Da|Kuea}j+0-> zF&_9vOlh(psH|JK^T$gz;;r~e0a~m_PDnf$0qm#eHsv>gfxq+p`UVb{5|0x8?BvD;4XaKTu}kVMO;CM(Gk8m~2YcaiOhyG=jG;H1 z&jQvrQLM*S5~)+9qM19TNx=et@jL?4joT=td>?f*t%9ax1-ES*0-uj@u-`O?^g(^# zJ*otXym%Hqtr{!;^hEK0d8l7`8|qI&KmODMzU$>k&WeR(TMwr3NM(aP97#N8zdF@& z4a<>vhjddHOIs+m_%8}s_ah=(hm&3tOEU8=h)d6e{@qUIn0Fo9A67xuXP#8q{v9&& z(TIL&g_77`G11^1`JHe@k8c$aiLNs*k2b0E`&4dddWV{8gP3;S0%Xsb3761%q>VX2 z1#4d*VE!v^u(Ti51O$=q_=}{hf66qjm0V|~W-;SFlX1%ka@(^9KJ@_DkP+@fo`Y*UjnX?{C_uCn*B5$`@)Yjp-))arG=Pbc7>D^ck=eY!ZNxbU-dDtAMY|mbl^5YJy86k(oRSg-NjbJV552<;@6AHQAm1Mf{ z+}?jT>}PsH<@=pk{&o|_hc|HTwaMIl-6w&47_rhtsR$Xdn}UB$K*oR3Xjy&-+LGgx zF}anTgRU{d(ZSqrP7BKWk0qJA7wI0HX1PHnwBh$Ms(F-vfDdn&;?f-!U=_=2D-XhP z@N_EreGoDpje);qSJ>-PDR$sH8f@WDjfzT2dHRf^Z%as~ap&$MkF$g3A!L)4K()@@ z1uiA%h8dyI%x{A4R$ug9lm*Qu1F3uADRK<`1(BL+DE%gpe8FzD^5ILeKY9yYo9>fl ztvAUnqNg^m@h8zO!>O(-_mlfC?TEY-N%A4*m?o{2{5~pK^s@^%;-aGV2gO_*e^M>4 zj#L+=6f${ADH>z?ljO!|6u6gBTIF%7mJ2M(w~?uSzRc_=eu1^z77~xCykn#b$qSvB zDEWWnwdV};YpCN!4(G^g(krPxeipLrC&-Y9HAPvBzT&;C-JN9#9}< z$p|P1_J#V_V~}sQA<>_crlvK2;_jDnc**)%==jZvG#4JLZOkemnqY;&HD91A;A)BU zuM{)-1u|c$NL#-d1v_-`$yHMG;!<8KevRh!Be+SVp1j)*P_^L)%9h>c?lA{YbVtFh zhFs@<%SSQouOqmk{u=MF8%BNwflS}tu6EWQ;pIoVQ0n(DV|&GlG5cxeXw zPcKDa*iwkP5eG{6mB(S89{xFV^aLe!+6#e8GBK~<#8QUizDc_5{nuk)f?@j2JyrKGIVLYj1 z9Ce<{gZ9G>(&ZKLG)EWuvX@}m`wm3rqovJ9<|94-J4#+ZC0XJ?sXFWpVxRs<+xK&_ z%dLSttOYYVqhPRE1HZH5xp(Rn3iYm`ShX7x*8fZu3%@{i(wl|}{_3_609*H*#Ua6I!lGoF+_MO;=F!J{I-LldcFO^41Sx9u?SwhT!bX-t&W zOR6~QU7BH$PKItE-L-t?H>W=p z)Qgcl#h1$480rd|NHg-JwDHD1wEvjS2CrRBj_*P49o1}1avBXTEhKrr@;v#LN}+DP zVn$;x(4tKTFl@;&49h7+x0-ko+g_KtM!!RS`*$8uXeaQ3aCmJzAoY))L?u^#ggm<^ zv|S6yw9tW?4n3y18+GKdN)6jjS5db$m9n>gqLf$nsrl;`Naj8g_{<4zyHP^UW&+Ec z5v6Ya;UFSDZAZxOzma^KD|wB`Vm_k-V0_{M6e=C7E;$YBYg%Zcr>I5FuXxp<6Yx^G zu(&S&VN6gslt=2>4#it)a6bjx-jm3Ad={^_Tnp>?Ua(pycyjJ_YWrO`N%ejyHOoaL z8tpPQ;H!v24j+K%eYrYUp`xVSMrgifMWHV;Av(TZ9g^}4W{;m?*Y*x5kF8+sDftlj z7vzPskA&mu|B))eZ==+B#8fDnE8+K8!LCCX-u> zz;VthDI_MFdgw1<@Y;NVe~m}-W#Nu17fPkZ8kChCC(nNces8#z#ZMJ@l;}^jGGRWK zA3MdglfH7*SQB3Hzk2e2vW!xOZ-iZVHN>~|kmTH8dXGZh`^!yeg0iKW?E}>E_DE^| zsW{Tx^{3pbI#Rx!#up9wMB1NXnMk}k*XY;D7?X3FMB_d3ywZD<;ge|E5PuuR@(#gM zyav7e8~1cNM0BK`(sV845+fnkvF8xo|1Qh_mP@rwkCCE&An-gl=wiM=6e4hy<%e01 z_i2z+AEu)7A$Ka+PED$)s z3bigaipwSmKJ$t&43iy4iBP{prhyL00guUbcrKUuZe@PS)i`35f#lD7;Ql0(RoFL@ z!PpVxH>DQJ3sy{VvyKOzkxn!t6E% z5vyh)U||z{x_u)bgM4l~cqSRFoj|f~6QpZWg+67ml&i)(ML^L3X6n9|{Cdygahg5Y z;&~hf`q3ocyMq_^s-VUn&ys##cd9uM4bjsMX~)cc(8WI@uYC{LnwnBlsQb~n&svDz zCQI#8u9I?52N@QaQ`IeJvb^w)sDA`BR+iH238%P8-Yw{Z1bzP3aF|DN7=Eu}dZ#dW zWlbjZ`aspz?-4b)liaFZ$>aNLWc1lal8|CvKX(r~>$Pm0EOlhI%wqli&O*=5wu$oUTS7C1)lQe|B()xR^c-d-fY`7Y4*G%UeAhQ*CEVT<&)5P<&H3Ww4ze;nv6ztlEmyJQ-1G9&ZC+r z_iG`VX9X~?)nQcNJqg8Ya$!7hIWKKgqevMC_g9x#{^NbL>+wwz?RlH0I&4VlLv_^c z(tEPs9Z&MPzevji|D?J7A5%bf8V^ZYO`0B`xTrD)q1z%6@+A==KHmlZs2B6JP>}5Y z60SOZmU$=JQCZ&-r0xhs>!lp>`f&!+Oq$L*o>(B}ax(dqe?^h?1j=hTzSoj+o~@wMBUn*y3*;&zsHpu3Q>^)o0{`rdjvwA5Jh*~dS2?0)kf58N z4u*Qo7tG61aGVgD)JfIwHZx0^SG8 zNtNva&0n9?@&}Km8s6JPb9?5%@TZGhK4|Y$^@U2RI#z)E`HhsgH4ZM@y-@V#B(r<^ zi4uPHCE3&N+%k@nCUpe0C%opCFNTnGa5^%Lh1`SlS}K_k0!g%prL45Bq;yJqtl{mQf{T1nHA86S3PRMiW)PAeVna@idjCz)nc==wkPs?Ieo4-?1V?9Vgt zn@Rtulh0V5M-tsEw&tAR4`2F-CeQa|dBhIt9=9m^>tr;&iGbsakK}VZThJe`sM$a! z%yCoE^-dlsWj}If=|qyq>XDtchyw=^f$ymq}8BcsmYMT*MWxkZ;!=v)d zW^ARdce0SZriw&{`^aqB21>ouO5P3MNS9=b%t0R@ITXtS&N-3k!&T_eMWN;TE3)_Z zr-aoy=wJSUnxuCm8#Pecwdw}hb}J-}%P!i{zZ0=1k5ad`%ZOVfcqT86c=4Xski2`v z1GY&h&|?j`cFUo1CrhN9$|ZGKJVmaqM#wJ0uBUrR)NCzvJNy&1yv#?#qgQ0wrH%%l z+5s=yn^N_yex$Y71M%!d$Q+eQvKdDZd?6CqB}(4D^BHA-75ulLXgIrcQg~G(8PENj zd+yScAKy*#ySL#pJC7wcN2AE<19Ncwf{vGqF}R>VGD>cse)&AY`lm=6G?P^>Yru}v z(bWFx5_NUTh0n&L6#e`bGv3>R_J6K3dUTDb`XrgmIzWY|FOkE-Fr;mdWX2Kp+&ii_ za+8EP^QLedHuO|C=IZ|vlqU+TL z(%ra1X}3&~-Cf{4+xAiXJteiR7BUF~c2K~ntB4VNXU{Vi5cA?0wGZe)X@kd7+PZjH zdEXZ_{4NSH@1TtE{uGiu4=#2s$iDpy8XH5ZT%sWH;mR4d4)#FWynYNRX6_o zkZc-asIg}(V%|L=BY7TGw#Ae6-k&HTb2C~uUqSi8Td+*o1fN~?tcto(VP!1sS`|#P zp?|9z;-A5CsvY_KdxDGVe^&>rUq@ztZxy&eA4F*c4RrF8TGICum-TzeL{-7O+A@^P zj+&Fb^d?WJ>&rHBPai{4?I}pVsG#dU z7>ZJ3ly+RC?5+by=^MaS`aCCHO+6j7%O}ZA3n*R(arZkJ+-=rrp;lg`3b!~)n%cl? zb56l(LjqfRyNVjEuE8xOk}5R9UHWq>9MKKR?&WZFk)mQp2bBcQgTbP9nC)|;>c8{g zKjl8;3qA;$izyD-sUh59a}B(AE=A(9YXZZ2McUMb6!*ss3h1Y2*448q+Q$^`3-&_o z)Sa3K{KX2>)ub8g%hO`dvDgclP}VF!!7xLpUWOn}R)pr|GB)hEkfHlImZU!81Wo+V zA$J$Y#5t{$u`2)(HdPeRcOV%K8$l(WW63@zfO=$Ug)E$&>R)JizR3evE&hqjKBbVq zwF!mydq{FgwR+^@GPHmCOi=^F;eCf=?(uARKRr&(;s0aN^OUe%{gm9!6`+&I9C_?neFxMxc3V1@CeBEtGpzBr!>a+lRC8Fuj1toFa_uI7()%N74TGH!>Bx zhIU0fFH$`g{IXK+H0>T`Y^f#rqJdIXjTti=_?ZQ)`I+jA#z2#@l%@RLjUoo`!3=da z)rW_W>*#N+-O+?(eVx<^o2HQD(_wU)e?TOjqTrzcHq3}8tM3!ZtM__xKXQvVOLkMi z;b?e1Z6SG4y43b_wxECSQqiARxxw-XSnDTJz-EDuyDN~4E0nbLI~?L)LH>J|)Y#!1 z^Y#uV#qlO)zFy!YzCTj*g%aWZ^ux+sIau&M33`F<*@fsVa|ZvtE<*P8>qt-$!h( z4p=u|qM8>%M(i&;QpWv=aIY&!i?~G=ng5YaDtOM_ny7tH0D4xsBeT@oyjodEA*FBOyEhUhBlU1wdkp?UPab2~ zf`Ife?h*bL_CAML@}PKB+07&M&^Zu|nw3}OT0)KU3ZW1fpucy3;J1v}1`!1T8`lDi0ce9RrvetgNij|u+y&*Mo| zzfUUbJA|intERd8KEtY2(7hv@AwF77#^?Ky=?~p0AkYf6m%5O<(@{!Hd;rHUMX-(R z#Dazsi25@LD~Deo<2Po^AY3fWHB-30a3rgL^dE~{e4O+f6aSx=kh|swQa(JS6J$d!ZmrD8v^O_Hwd>*GFdtnBbjJzadS;ENGl9d}|Iq47;hV6%D`=6{PE0Rd?vK2Gl@cOnmZfez+3@X1- zHam`xwm+%=Unk%lycL$}a6u>9p;>DQhpNY-(eYu!gqi!IeyAwoxT%^gm*Hku|i+uCwQ?cL@^Mbm^l z%sVsMUK2<4rWZ&ry@sN97nqTu6$*dK6PKdj?Te(MRm^1;1C4(&HsOMnoa8zJo0KoIprX7FA!+LHkD{lN=vTX%%34MYiBQ zoywKoy-aCm0->C6Maa+?F;hhjIpl{6UgHL)`O%&g&#R}_zCDH9u_abs@+I;84Xk75 zKB~2OMzZ56JZ+Sa`Qpcg67IT2c?QYJat4<%h+ z2=}17;PXnkrtBGyoX`g~f7em2D2tX(e?t+gZOMJ&8u+saGzR?umo9Tjvr;8BlFdPE zoj0vH^O5X7m-5!~dE}p+h>Wvip^81iYU6|)U%Kh+D7uPUX=Y;9&K*S zCi79pDCFTtI5pgY(z6?L6^~=$mSsE{pD1743Dfs;Q8ex;IXAvz#l~sqxD<;D&sW&t zq(`OT`5XWHhzB0;fm*+b*p=`^;FDuuPlNacJb}yfdKBX;Ss&3+ z%a4`Rlyetj%0sDQ+$HD^&&Bq8U*WYSklN$atan2hnWo+se318)zb*EQAG~f1!9*tbcGkz?zxM;8RyI<=xhn1%f0JCNh{*6(6;uB7f!gM#Q}((UFiJ0l zw|X9B|GXc%rXuPXI+x-{Xi#)%4_7)bLxuhXscwx#_Q4vic{D-o+9^f*%C3lflm_v% z4{FQlztD~-f~e|wp2hmdw6s2%e7~MTRn#^l-TI3xJhI?@Gyq9Uy&x{AkVZ`13a!H% z-aqWA@INvTxJ4;TeIyq=s0B#b+JJ~+7aU1RglxSdR~e+LO)X7Gy6idX&t0c>y(jE* z=cC}}TNr!&%>%@q6uqq-Hm@=S-Iy~Ky-rcohAcGylZ@~g3UUqZ!$q&+^4!XrsA&6J zln+}1ktR5Mv!@OrSLN`zycUKJ7D4oU0<9TWO7-;&n$TZ(&D9_z=VX)eos3ybyAF-u z-6)1-kdcoWj8}bwOAm7)-<8Zm?V{l6a}L>NQ=$D-%mRhkbL8pwkaevky;jhi|K7m% zod?leI2YRG0qzm$`FmsWw=k;Tww7f4fu+0p85yugWo_GZw*Th}R& z&xLJA6cVC^9gGD(t3|SJ_R%?&+{-2k+mAhk*|&*Ql(Z1)MFiD9LwUu8F!)8bLn7|S z?RXOybA~;rlW8X!{c_iFG&CJ&Zn(d})L_@R_0|kwq|9%oqB??#EDVTfvh1rqO~4&*5jD z3gaiwxy_0oL5IG}Q}yo0CC}ZVpE{n$Itw$_mjN(dH-^Mv$B@6K7HOJ5D*4hAx>I?q z`5lu6?(Bm2=fX@kIGTx0bdd40|MAwbk#OD>NUq<*c*5HUh!`UH!-E5- zrkOJ46SkS0XGF5D1B9A1(gB46=d`jB_JdyC;8sR4JS}(&Y{%d$z{uPv@|}Vkp;)0NlTP!m;OXeb=SzYNbo@p-+}Ed zL5CbX%uRpVya2ykR+`W@|+#Z#>3{@8#rct zM{d`LaJP8NqSk+fdzV74D%huv&|RkXPj1NQ;f$0&79!;TvxOU*1yp|=gu}qokc}}_EBDXlvWZ+BFsFk;K1~*~ zSNZ7nJs5UV8puz`*@R4d3;m(HWNbH|8ypW4@*;2Z5*Dtabf$-0LkP^)7xJ=e&&cOj z1*QGw&Q&Ep!Mj=T#Y=>|)BkqzgO~1-*?%vYT-aeVo}9|7H&&3n%9-SY(x)nW29T=o zyx=#iV;yZl7z>S|WWWQFcLHBZQXyy+uw&xJf~^#FJ&O37A~4rStJCfl`v$rrts zT5U<$E|1t`h>?SeNR_ezAM#jVs zBzitvEqdo8ZMPl|{SabJ$BSUR#GOmdCvowp40Zj37?>Z_Q%3u3#15HBi^7GTJ}rfe zTd3U~zEhOonRx#%mWmcy(@0S{r9JHrztx^3ckjm%Kb)k7 zo*`J!JC`hOlvAAB2TIjeLuK(7nl@b}pU@7jwJktK(_o~}tcUTg3CAbX{b`aY6%Ej&iw!4V-OUON%bA@CY z6(`q`bW0ui4_pibqgW(8%;cGW--mkJb+Q^}Ks|zC$kBBW8an#jXlPTPIIkvN>x@|ka^)&wPV4?h^q0(rZF*$>xEH8(@``0aevYwTd>flvsi9m2V=O-mQ>D zx|6s6B--374;Wkq$@B4KRL7{fD3|m`JB2!Xi%V=5Q&)Wj`5O!*-$5hbuti4(Ra?<} zMLxL{{|Bj&aK^9o&`U0H>&cPu>pqBw%-=;y%T!*=7b9J+fWeMh$cFBg25y`VhvgUG zWwMx5?IWoC{07L^eB%MD`oQ(k?}+{!%fG6$ePWr{-fs><6El(cpWw+HDC8CU^Scx--Wtrison{@W;aOW+Juqo)bM&enS4uU zAZfvKZhZ1Ilk7F*_Ae))q%4eTJE|bRd4=1$bl{8mK@z>scB_<`co?PNJ{8A(0{F|Ye`$t-FcZSgol zwplt@PANzImsB!xPC|ZIEA;<4lQPW-3*?>D!%~a6;(erj^o&+cj-!J?pQ(+%MEEjIygB6LA(Af`TbB$O8ZT`$)!lBm-%G> z@ia?wGiQ?T_NcaxCjVg@NS5=7MBV?dqO%W+F?-|qNRqZCBuN-HNm@oiYR=uPP1+=F z+9WMWOK3@wBz8)YBqNn18A+04RFax=HA(kl2m@@=eoN5<)X~;jfb@^vAd_nl-*t@V3dyz9ajXW>qiE<)8iYZaAsYS}4rP-m;}ioK z-zq};fBBF*6pR)J*dAp1Yw%;|mG+{yjHP-*_3iYqY@7uBpDjhv;x;mCdl-8D(+p+1 z3_W3_K#XO}Bhcr;kaE6Q>YL6i`aZ|sGI0FmVp|U$+u*wJBc`g)WSI(tvL*=(NKc_~vN5TTXLIi-RQRykXSfBI z43|Mf9S>s-T=8)Gb>woLp<|;0vUNhiqC6Z*jt1bs$@?ht(dDO3nvPg1Vm-@Pkliw+ zb$1HT`N3Z37uh0j@<_&mnxy)V>m5TNHzUr=LT zHAL4ufoSVEaBME4f)~GGZjCYO$ESem^akj;S`Fzfjg&ub5qLC>XHgswz&cFgBme*(lZbqO=S$8Hgzp~47yQ~M3!z2 zvP2~*YpY?qz%A(PKOH&qmq|I{dkaOkFQc8!H4s-ClHl{JAhmTY=s)U3kHM29W7H?) zZGS^NCbdIVWf9Thd$an$2_g>Mrn0m{e8*EpL_56@(yMpF*vi)sJL@Dmrkp|aO-0#a z)(@@PfWeP5*!gjTI>K|5M2$p_bxMI&uPwws{enu}dgT9h97J_d)Z<4Ug;>gVkY`|P zMHQQOPQt*5E^zy`6MMWf(fyax;K&#{1Dn4fd2%>5R@b4{)O4c1I|72$SnmIu6Sd83 zC;n>(q28pDczpZD*Ph{zjt9J`G=CFxd{%(b>LYW( zA}R@d{?)^D9zeJ2RVv-fhYImEFwUz3kMl}a+y`)g^!%iw+MlteD z5@hZFjMDqZaKdcnr0{(YZElfZTYZ5sp}i0_XCiXW-JzI$6&!hMsK<3vYOC5rjL-By zkV+v$t-6K2?>R&{7)8qpm85(0apHSnIds)0ps*&BI&Kdmwv+Clk>5KKc&ix$Kc-OO zu1=~r7Y@SkMC9!hV2@fFsJpQF;iE24H@HHD{hGvd*D3U5yFdwk0CVnbXwg~=>6cGo ztbZV;t4gUojRx7lxop1cLp_>Qz~rqJh?f-8N^%B$p0+~5j7ZG-tqJUYR^WE8^M)Gz#KD;BZ73@E0bZ+~Ldx7+^!2>NxFsFzi*EEf)$PBd90L;Xje81MT5vfI8u`kT8Dlh=mo$(NBcM?qZAbRuVDvNWv4 z5UM`RK=u0);;TK5$Xl&ZczYl5t$jh97Sv(6N+)P6VEZtZNy~TrLWR~_q#TSYtSU5P z9JD&IzBL)lUrd2rD@5SCyqH+8o)21!k5l<>HH_8p0hNqLAbfe9av+YA<9jyn*?NCu#b#kKo02^$p@)2;RE~72K^farI5qZi+63p&r9MJA@nE3b_%e-BH!;EX5 zzkuaxpLMgoQw=;$ETbBkj0uR|g7s!Muzau{;z#UdT&Ll* zY;5~2A0y)R(B)AnsMoHC&VR|CfG=EwIM5S4QjQ@i~EOEr3$5+60$t$YT6^TaG>>WE#WZ7*{juvy+T+TRxDCPAu zcGYVT8jO>QmiZBlXIH^{+zrAxI?J-mr~^t8OHfn)5GbCHfb^nd+Tu9}B=_bJp7}-g z8M=XXLLur*c?Y)b4?*$WjT$cvftb;y(iKl^p%jE`Z8`PZ)btA9af!Ls`Z_OtLFN1<4_a0mGoakdF!cW(>7F z4S}btp;#>yxmQE5q$B`rm$VTzc?ss{vwWF*A?c2CAUf78x4Sx$F<>sF;pS}+RenO* z*_%Wbw?$e$kb=H`xilY)1RvV~)j)KTHr!eVDH#+SoK#h%WQOW9u&dZV5QgaE?zqV2r@_z@& zY0}S|h%%4q(tdLd=ve-Zt(|)@#;z0UJts3h{SoyYBLZ%lEzSHo8KRctp~N;8YhL z*9u;9*?ukQ0}TG01>!n)@cI4_9j+9Eh3PO#1& z#JY%pZ!p{2KHNY&-cBVu=Eb6N>tmvpREssPB1l-y&d(82aJQ)e!<}bA?{E%wEYAkc z#IKfu|N5}}csXoTzK5zgTOcLw3CaydF`o>dnjdTfcf(g`^7Cy}E(-(aa$}aQn2NyW zhnzZB8vR5J6_Z#G|L#v|m3us@PSJ+c3o?jZpNPsy?S%6(8eE4()OBYy3M-Cc{IPGy zY5Y@~6nhuMzfPyAyYIu`s3)ja+6zY4#-iNq2D)#p0N=7#RP=EbRUGsI5AGf+*i~lKx6W`Y%2^2p0kndYg?fy8421oZo{@z?PJv4+~qg=43 z{VtpPWTCm{U*Kz^B*Q94K-aQ`^gA_>zB(!BcAfiEMJALr{198P7~p*-^6xTIgsqviFt)ms2_V2c$*&3`j}xuxXbN4x{?_5Z=Z_AcN%vb;p^L@3v41xMjow#&Ol zGM|_OXW2L+yX?-KZ$6-XXD?%*&V&A?+YtZq9Z2qu2Ys7as4q4Gi=+tFL*$|FZ~}@G zN-*2y1Wv#H5lS47vc7K|-B_ECM*AnQY|$MeTvTnDRi1|0v%f>2-f`kNuLN5{S)ZGq zO|=~*%=?s&9@A#gw1{J9`|>(!9`y!=lT4aACIw}Qh5R(_0bDXz4r$Kqxa<2FH2+#j z8xMcRsF|-ZZ`uP4oRNa|{<`3IlJ%FvY^I*z3;YZ(tO+Xwq2+gKeZvztYNJ7L?pMaJ zY#_C7{J|;uGV~9=C0eijAa;sAt1ljdyz(Z7)i54H`pnDStg-vOeIa7yah9p?}0?)B?)D_@{QF}5NJOG?!uQ+w~Y7T zV)YIslV=l;aUsM}^E>f)8~{eY{t7O4KEnLWQ!INgBQ0v{!0388Dq^g$JfR!~s|boh zBGAEw<(7PY%0_KQCHD$&J~bB>*WN(&NS3Rxx`dpeIiPdv4ampMhS=1R?3sLo zzVH4Zk`G%!V35z(xHN$BrHn&q|AM@qGf42pUqI(XC=BE!gZ8%T=>AzBQm?UW>P$Dp z<3CV7ay)jtZqJppz)6Np$kLk{DLRUFLE8UX4uo*aYF3x zpNq@mD$!^6LB?!+1V=j|4Lci$vOk6)f7$OSUQtfVy)tp*;8RE+d!0C(XZe$&Zpdpt z4<){qFl;mgf(P|PAl#4IpKVZ+`2(^hMWg1{tFUD3DOB5D4c=qQX=c|w7&bhGRh^R{ zlb;7#`@4v4au|)>bsjap?ZSwT70fgC41$8#=jVJN8dqLGiQ&&Ac-h~`iRXbqn-xmM z>R5KD1!`UjQPYCCsw_^D!++nyLCcvqL7#aK=dt%k*ivG2#v8(#Mq=)|J`6Bp?^%B} zYHTP3-t0lzJ7EL+Tvihi<1n}`o}f6BO3JE^B11H2xzTelM?R>8Uc;2m$Ba!l4zf)r zQFuy=$aNQh$E&qawKA56JC$`8vl5T*y@gGg;HI;6bAvPS_}`!*lD zkEc?5M+G)MLzFC?!I+P&ERP>SWuZxgYcd0ryYj*DdKguBwMcgz?ZT!@eYiaP2DZep z4AC6Mum+fdTi(v41zD*7tI;kLS z83&!EtQL8E9jkOt6aQtNpp$tO-CoIHY-}>~`f2hjtA5~NjWRTE$c1h*Ya&!lr+!;f zQE+M(4!1vt1iF_o2s25s6Wbl^iDNr#6SQUhtn^he+MR2H^2#fqo0CC!LFG6rjq!T8 z4AuQosL9UfpjfV;2Ff1reYKYM*|55yu#MIp>B3nPl0e+!2N5a-#CDDX`v(m`viUR! zp1%U^>~$!h`q?$}d#pD%iFFl6Hr zLyW%~jV3osp zzC|<3SUd)CoP^)cW+=T?Sx)B{48@I?XIOKpK~e;JjV`7mGn z1EQf8_%$YZ?UGmIC6Q%XXfx&q>9DcR^3*JtibE*GeryI`?Ok-g7{ax(50_ zoCE2{A7DgO2uV#C2?i$Y%ovz0HT!&!Wh_5QhfkG4?!jwN(ZX`XQ_E<$-a`;NucnIg z`p6|5YUKK!D$-3+E4GRB<-GtS-&msm`4lX__5cFUvh$Cfm;K6kDn6*v*#*Ab3b5{F;W(?DOk~k}aLkKO+^*ydHz%+$LgsX&Brud}djj18WLU>bZRZf4x(&40i5O_)W z6-${bu>KBnMEr_1n=+Z(?;^V9&qm&Y7O34;3cQLZ#AjJ5s{N6OJy`?LjfV)YZ5zqZ zK8*%Bq1cjh5Ja~YP_62}D4!pU2{{z)O|M|u=s_%Fd{v-BJQ2B{B5F%ykbhAK>hW>Z zKZLzAc8}y|M}0%-k*CmC%lxta_t5b~Cw2Y&jcoN`GsB%jxa-VER3>DgEGiC@v%dpp z)NrAo@-9TVYeCDII8ZML!~S<0p}T$=^z(Tz%S?u%9XuTNdkMVp3TiQM7v1mIqqycN zP5)0`yuP?Fw0}-^3#{JQ61q2(3*P;+_Odl_p1)N$fVdZgUkFaYkfpOA<63lz_)yFZuF61gI@PjPaMkAk1SKxGCyk ze$@+1+rnoQwTwC z{Vgnc{0|8K+`<}O_^!;lTaD!aI z{bd~FN93XSZU_ppvuSFRJ&0W#*gnS(yCXi)T=9GmFHYx|oZw)Rx(W=AkzwY!FQ}6( zg2uiU5FU$>V%AZRTjoO4$ybaUolX>6tZ4MvFX*xMC+e#{9)-{MO5Hy6g4Ed%4K6-q z{l5}8(~9|xBhpauC5JC_8Y$Hep9}HPols#Ng|$Dk`exxM+SGdz^1icK$EsA8`SM56 zB6aF6c!pXPmNdnn7<@Y4!Ps+m!7(U}wnPP^*0Dz*VZK0a=_$UBMDQ)w3kG%ew;Qik* zlzyCmou#Y6X>T7?8SDcG=8LTR?JiW^SVO%BEFk941q?p&5%|8BnNy5;9%Jf29J!K2 zZz%=6^aPOYO{7)6Aw;Ha!jH(fLiL_Bq2Fy0lw|gz$ukQuT48~_cO!_v^chWF5QF|a z0X7u3VQq2(mdV4waNRS|UHBOzQrao^{8GaAk7kZ9N_;IhQ`foIhiIgHd(c1bPu#Vz0307A6ZOx`0o*wUbz7UEK9G-o8q8lekG%(k zGaztmGvV#=#PE!FkbJ5JjICH+m4_Xu|r-v=Avsw_nA`fJ_K%v_R>o zo7nxkDis{QfE;@@IBb#!OTMwX7FbUtzeUAm;e5YT#!+cM0`KrhR2g#>#u~;#_LLH| z-p}^nrG21&`y-VM$swtw2Na2nhtOQeZwaUc{4QbMozGA=Hx=zK&Ihng0;kk86fVpn z$;=HGA z=ImM;<|sPgO}YZFfl{ys)rw!B{BvTfzL5G1j+cWA~kz zz+s@W+x?56SoNB(uUHF$G4sG;(J3sCWlpRecZpWYS(@J}BH|_Q!04$Ix>M_+;L2%K z1`W`BP74wId4igk%Bc1w8Pqv3ADqrkw4Hi}#3t>)`dz1S$&CtZ49WzF?tkd}uaFut z{=w(a7Zl2$7rJiu!BB0+?za6$f{#suq={)z5X#&!8FN6;tSv3xbPR*vjzeqRb~N4H zf!ag&v2H;QgoUq%zLE~G5K2I6O*UiApApTYA;7)86jY)gqv3~gwEpu6#D<7L{SQ^< zB8x%Y6AI;H_PNM*p%tP2q+<6Is@91FNI1eP;W&g{bH&2eE4l+1Q){ zu(ce6S`$DS^B+#|zlc#I?a^PGWxdwwf^F+{uKDGm{x^Hjamy6i zE4_@xjj@m#JQ;OHl%eg_Oww&QNHkxoV`$Jl^lsP!T&W4Vu5m)iG`5f6vFD)dGzhO} zTB_&1raN9_(m{pQ8|)Hk5ts+agbii1r3U1+e= z1ri+ogjkIbklP!ho!u#j4`ca^l|d-mypC$Ewx&Bug&>goQ_(vmvHjzBRF0kr7Mst4 zf>S{pPW3}nTr&*z-o~yox6oo&1dGm}dT9j_Y#Z z{G&y^=dtfG`vjVg*FjD*%Le^441(kUzMFuZarcF&T{D}p<4@5MU(w>&E7-kgB$|#c zLRsrxsYBvT%uTlkMdW89iJnO97bk*y<$mBkYLyCCPUnjr+kv3`6}sLiB?6pH1oD+q z(aJGska-9V^nQU{3H$R)ZD4Co4SKVFqiQ9e^hWGMXT5*X>#`>L*Zqs)HJYf@dQVM_ zE8+ilI>d~93xfTRsH6NL1}<%<)>|VX^Z;XvqboqNaT(1Be#q_}P{Eo*Z_)4$o0T~e z(qx>E<%<6cAVQsG4FB(dZ@mwihFJX1_H4m-`TGp$ZcJG}`g! z1@M<_0o4T(##gi2zhewBux7cZp|wPHmoAz-_r?@?DR^y~1qNlQP_mQ#-P8UAAKMD# zc>9y(m(DSdycWj4yAOd*dr-B%3sXk_fcRNgAo*Q2lyA8J6LMdo^oLp`W3G*Nnme2jV5>gWz9Vsp&tp*y9_@oSC)gdXNi3<2!7>fc)q;dCUQ|0teUV zpvjAWG50dd|9b5v1I#nvd~q$h-F%G(do{sxe=Fq6zN5850h-PE(78$x=vF6Ekb4L8?c_#iO#Qsz;>l42C+Plc&{gz@BBh_MXRv8 z%7uC;mM|A*7dTJ*go9xVu-KDNA@uyTrry;l*V!ehDKnt@+R=Aa)}T;mL2`R?NoakvNc z+irmDY$!jqI16Piwo-5XGpycz3rcle6sAlhKJl)Qsci}a-PPD=bp-=Q+$3UkDe~5? z1Zm!A2wS@nn+}y@L{}2&_z;awSzkcK`3jiSG5>wYZjg&tLf>5010r*q{#H$AImLqK z*!LiOUqY%v?xNVcSeiJ2xs&%+VTLjfbT|BloDAmReHM$>Gn_%tIEHHd*h0(X%nQf% zWUf8WLH$cLRkhp(Eg#M@X6-0)m}0(r;uq>X-4TtBvCQ1rO;kG53B`lkX_{jz=r>nm z_l^uI`&CTc|2P1d>d(NTNR0Wpl!pG+gV|=3*FY+sf|YYwM(7aBNxT|{y7Bj*jQzYN?Jt0rREk;u z4FD(adVy%&c2p0Y1nK`A2hLI%Kl$@*;6;z8o-yB<)9?xid(T)JWSnAiGuD5TtI%mD z0BelJlz|V(^^K4ky+LBOgK>R*qp?0mP8o}a>EVpIiw#G{xAWbHhoE!QdCqCk=@;~ z2WxD4(KT>8MufH#ueDQ9E4vSM2STXN=XNlEuOQyT^FZfj2}b)pK!>^4(3~+K;%`1Q zHh2Rnr!uzP`!WRI;b6(N_vkUv5p92e4R(Q5sGqSE1^>8#_uWn^xBLX$f~h#DaR+wj zM?z(tlHCE&2i>t^v@lPC=mVFq#F(J;nj>)V4%ITv#H~XT=nmHb0hz?#75EHHCai>( zos2>7n6!*<(}39%<3Q5=7%L_%WA`6?Dij>=ltz_o!NA6NjL&_I<_}o@<8T@^_?mz* z%iltH;!SXDlaoO*hMk>9sdE4XvOf$i$um}gWd z@sL_nR$$M+ zgPZ?h>8fTZIGG9cyK+zxLW$O|{h0b|92m6>Q_h1CF!)%&xVvJoeRrIsBuY^}{u`P* ztVc&jmI=2SgAoJQD0gBQ+p#T?=EF9~H}^qZ;R`DG{12VJw-7uG8l}3aGckGS0*FqQ zvbx6!Qa>;!tHxiLKj$DASDk^RPcy-Mf(~%V9?MGQQxpXy60SuzX}o+2`s*S{aqSRj znQf<08p(i%n$YptIjYWPIULC}i(O+|*gZ{qFw3PJf;ayOl`D%uI>#CGwHQm}ABhGo z7GSQ{gH=llQSZP<3~xNkJQ_2g@8U)5-mAxI={H~&B4T%|+{ER-KVUhYNLpjv0+O2* zM7UQ#wdya?di9?$Q{!)zf$4^Pu?^}MEyf7Z0?hAy1I8*z2>xsa^fBF1xYz}qdXu4> z_Z&nyGcZDa8cdwe;-H@^4zEv#lH&JlPF9b#-B*xn!`w8VcvO6OBdw`SfikBdViZ@5 zRkJ2g)&6+KiLN0LYi5D-=nkyFAK=88jmQs-6+gZOciei*=Eb8)jVp7LoDV>?eeckH z^HeZ>D#TeWpP=7S3%YH>F=i+qJd~wWD>wrW_dG?dlpW0T{}%*iDv2~-A3F;cV$0$Y zX!s)&!=A93&!HI=fBZ{%TcZdkVS=S>M2^&a+GDggW*+=i(YV8^9V+JNqs@~$Xei3Y zQf4eL-+eivh3@sKZOx%~62-pHFl!}fjzf5XdQ+xdkICaeP=?jr1Evt*;c zFJRwqap*Fd?fC9STAIzM!rnibFU<94>aQmPty%ksV?;g??QN%=W7UNc{j*?@YzXP+ zjnVPx4^aI%7b@;A0NZ)~U=ws5WW@=zYEv$$pD=^5NnDooyU4FCzK+Hr%^+&cVE0F) zV*a#f+A{qr$OMPb<4+Chvf|YL^Um1O_5t!nMnZk332>@jEX(>Q9@KYeVU^7S;unYD z^&lD*c4LSjoIQ(`i%C~i0&;dsiQ~JMG-lf?K9 z-U-cH-;#X4Cp0kW7-OE5g@em?f%9r>^qTGqp1(waR=76NeL5NQCto02n>(>eeFpXZ zRs(K)))Nc|NIf?mVa&%1((_3H{i}o^v)AWW?9c~p$84hh;47`@DMV+3y=d}!JqDhO zL%*&*a9GxWg89{a?)g@JJ!5t_VZl^hX91mOb1+cF7!;QprM0<%%nQMC`tw)Ps?)o% zHjW2LEFY+)Qw$pa?{88KN#2GmobLXW)sg`qHJJ~>lv>c86@iUqBFMeFhV3TqA@AT^ zlCVh|&GfukuJRbxYF+{PrMqZwS_J}kWI$D%E|%Z{5 zc@MqoiZJQYeb{(A4P$DbfOO(q^f-QvPVi=quO&XDC2$qC-0VWHrIzxUvD4QjxWXOno4$$PBrw0-J?t93}Q+Nb69_x18UzH?>?@A zw0OA!=iLZe9x@1u^^_`}?}EgFeiRyiL}8 zOQo(|x{&|ZJ(8-k72R+C4Ia1MiTjP)$g$bUA1h$LA2pWp=4yhb8JoVdY}nrSDEsCO9yMmncb!1IrUZh(dn&)CX&AhZt_EYqM|Ci6 z%Vts=#w^Z4$+rQT`P~W~huF_?zYD0JE+%5#2x^eO0_+cHATPs+1ZJ#9yB!gbwX7T* ze=j0+9hJ~K-xy2`wV0QOvYg*z!n;;QBwen=z4a*!8(co zeNQHlY-t^IX&r^CtqX|Lt}5W_nc-M?3iapTL(s>5=CWAct0*e^AFpgB=#+)v0BKdI0J>_G78NXq{88|kq{L2 z7^1Fruo)$Dy+54J_5mVvVf}uSUl&wKYcctwf~hYb;dIAK(6Q?t=4(4Z)rkjadt8;V z!0W+jAFEy0@1lLLZer4eX0R2u(c-U#%whP5wlo4{A4MDvLJ&PoC0a9|VFk^AdY4(C ztUd?|&tkE*Z`rE)b4^L&m?rz4o(w3b$c{Rin&<-}nN z%WT)VW4d__?Ku?%;=+Y&m)y-5>Gx=6bQN8n-zSa7T2Y)=P2&xIpk&=IDF3(&i~T>~ z*eS2Ud(S8^_^}5w_4=Uqzvs~X!)365!6XBD}c22=Db@)W~iIpmii@j!}TI z$u;Es*vU^A*-27IS%YTXc_?3f0gU6mGX8D>2w$XH3i7t`NGU#C$#i~DaF|v_8TStndoVW$F;r3THCrCz)+GM)JbQmH!&#@Yg?fd7ckxbU3 z1eu01ezA#~A5JFT%n>lC5s$(GBYw>Y#x>~7L-DIz>T%>25d=Mf8Xv~V&3Xnr>2hK$ zEQQ$FjQ#5npyn@&aNt}5nxtx=<6tJuJai0=#&r|((S~5FeGXkpdQehfPlaoq6-xL| zh-+pv)GJS7OSKA?J&$D`qLJuhT8k<9{h-`5gkfh_g5rQa5uV`kGpjaW%kd=eb-Bpy z$||D$U%TP(wNI!UlTNiC^yW$e49n?kAnMH9nAdm$jEWnnt0xDzCx1Xy^%UsX zTnJ7zAs{epmUgF`Q)sQhg2xred$R{*X9fw^%8CeHK7)YvA85BWAFLyTz?XggyDz^Y zwdGU5ZD|d*=zD`^q6ynEZ$aL=pJ;8a0JQv%0e`3u#ebdW2RL`J{b?^zm)xhmmxc83 z{#w*@I|=rTPY}e0(T>+_S9c{3Y>$=`*(ocMFuMr27{u;O$pM?IMabiMQu}qsnZs}% zG)(!7EkBKCcgT$*fp>>NnrefdYtOU0InQITg*SG`M^i_GB2ZjRkb1odVKa^?g%)p^ zb86b%LOYLZcvvoHobwAXjEqH&-4JOh)j-{s^Kj$9TL@!)wsU7OxW4H{4*NBNeW|do zm9NL(%dlvyRh)pFDJLnD>BGi+CEYMX)vE5?04?9xEB&xOAGh_KprdkYAZn^RhQU3w z?5eZm{-jp4Jmik$i7~_UgUNA=dA}Ef;=v0`m7ff(g7+VSV!>>9D?bYkLve*aro7_Miey9IE%&~-4uzu| zpyI?5&>RYZBR5w=L(l>`@<1g0_4!X$Y0?$6SiT<)&5VL4MmhBFg;&VHyUl#EL?~Uc zs+!hWf1(o~&9svISPSRP8>Cmi*3$FEV=!#QUB3DXb|+|HFa7*vI>^6-q2;Pqe2>|D zy6>V%VQ2&w{rzW4e=qF9!Cxm>eX`U7(~=Q5`&b2eecKc!9iB#WW}IWrpF7~FuW2=3 zU(0GF`#va;zi;-$Hk1k64XC5Z4{u?#b#=_Rp>xCmF2}OyvDwNy$ zj?hgVL|b-|jLw-3nPffLp+13#e@61Ku*>|0z^mwe?%#zj;Zouiq6%3pBAmV{l9tT- zo6MA%lc@d>Y5jyLc;oLtOkFStIQ1>6vVX&DuSstUtZNBB2N6|^WL22}zPd}cO~ps73H>!@$!D(rY+9Qs literal 0 HcmV?d00001 diff --git a/tests/data/glm_ptuning_v2_model.data b/tests/data/glm_ptuning_v2_model.data new file mode 100644 index 0000000000000000000000000000000000000000..c7fc1549f40ed9713686246d5e193572371f0b80 GIT binary patch literal 53012 zcmW)ne_YG^`~N2)PA4Ix6DFNbl1wV8JugENCr&zXhB%WDLK3HUl1UMgsU)cksrA#; z`l0r`Op>W&l2j@~GD(t3l6+sE@Bh1X>$Sa}*Y)^ufA;6^;{+F)Tq~O!IXg3Zc|l)3 zwVo(@T=3MQBj=E8qTTZaNt~(qYs?c1#?9}_D$ky2JHhr*PCx;_AmY~3!eQo13$ICE z+BV2OTOG7oIya&)*tWOuzwGA1NKUBTPTTi|+oi)LMlzqm3t8@*X4|{AOXjB7y|gj4 zT_p|VWZ3>I8*3LZYwP@Jb}F0i^T*6vmyMa5tgqO*=M2au=OxQ}uV<%!w92xm$j_c5 z&*tU%-!xfpJ0~dP_ggz<^X;}S;9FW3j+y&aRwWHD*l(*>xV2#Nf|*uU*2D8w%s(*S z)-Eiq*-j*_uzO>=UuWHVKb({aQ?EKW=WS-?5&5=9fkAr!xv-}T(q2! zf6GpD^>X24>weScnO^y2g_{cE@_ZKTF%Ow(X1i#%Gy}2~Rt0t!roW%7mES*i`MmyI zor1`Mj(q(E!wZcI#@@O!@6YTU*_G^VbDZs_EI4UrQTV4UsSu?Nc`g@9lykuJpPWzDAMzveciUdI)svhnxLHW&owWIQD`fsU zn_B6Ge1+{rt4DVAoP8Fb7fiK1pSg zFE8-3dzt_H0&%_`C*Rhgz}VJF+H1YLu*>ew{JOkZ^Df`&T_9VqHP6a!xrA-~uPw

YB`Y-FZ=GkmVxFt5 z^MYUVTWkW@u32OA+GQE@ON0ARQt$^<)y#ta?_Mx%K|BOZxeu-uso=at2=!BTpu6TK zxKB`^?ut>M+WmoSn4AvfzYRf8HHD(_1JrnLCJv^5z`&so&{WcbcFP-(_h6%(_jDQ8 zd}9{X;%A{SybG+pTmpU>7b0#PqqXzyq3%>2?El;X{F5!j=urjh|6? z;u1Ibuo5|qd|LnJJ0{d+U{m`Lgw4B)WpBjbo!yL*OE0ni;Z;;ldyE0GR@k-VBsFzC zhzU*4QSqlO<^Ok=;{A3M#zs2|#4Zve-~)DfUBeIJ zx}RkzS{I3>7Ud|u7KO_6aOyQ$4s3}W*^^%(hi?NWBp+1Y7816f8*U$#4LuK+BKQ4M z5RWh+=Z2NY6pdXG*o=76bJB{;i(M*ZsNL@-=N#SbnZe{2;E>(ij({8;MX zbsQobE`a;11n4na24$PRVyjpU7WY4*+oy*_XV+$6+fM<5#9`>8--g|{W1vTzjUGpC zfa@?btQuSniQ8?^WqB$Tb(9j8y**`rOvk~KW6)zuB?uRelk+Z&M%J#Ia<$SFMV9%* zl&=D#*T+Ebf3Kl_!zmD+Z7dluD8%}d*C=hiOKKNqf%k<^5GLyZwf!uxT$+u#eYcTy zA&_vY>Zm$cm$-fQr4j5@YP@AUYL4WA_`n+~*t8rua}N<-5QiJJzW`mHtq0kyeHfg4 z76T7{MbD!;VEG{*atdQn@phG*HFp_T5%CWhx?Bv3-)G3Hecyrhkx}5HZv*V?*+e&V z1%&u!LtSwRCf2Qnt}mZ4EabX5td zaKcr2(f4MO5LS!ht4?9y+ECCx+(g*Z66EzsB5YYVgo(wIKt>*zCKH#1HDG%hY1m56wOxza_ zq1k27wlNoY&#Ry*GZYNu$3gGkXQ-;lC#p#gz$dngbPS!uh$Xi%XmTA`93B8gfd!-l zZA70%_t5Zt4vFt+q0am^RKKf3!K-w6o!}9OI8z9RHG>KkeIbUnpI}H>53TE?knej6 ze5%q(k;5PuEVu{pmyV){WsY6n+6ilX1+=>5K&&qp+X~vjc=q40p&=i(>%{|a_%Fo8 zet@37+5+uWi5N0E2fe*s1Aki*1|MGwk=IR7TzH?$iqLjcBz}U}7fW#DyC`hUv;pDc z$(Z`#DY`r|LHXJ^uv}n_^&Y=tye1D?AAL??J!(<=7&=atV5Cn4u*O|1;k-3LKe;`2ie3Or`Ggq@ z8}QIqHF|w}iV@L^fE{rGU7Es3{lw#7Ij+&H*voZ;9wl88R`a@73pglw-)!ZK=h5GAUK zyfG9NF#z$?KH$17QNSOOPO^qFpvLqp1Yg$zwK$J5LW*`-yTcf01L4UdAl)*W$nugg zp0^Dw24W#7AQD)Y|8lf3Z34SA!v?Jb9}PgB9N=s#4= zsG`EMP`RkLlp0UXM)j4cm}tHXe8ba#A9s~1ZWnSp{RW`px)L=tFKDCb2kaU_=okgnF|FAShc*oxjH;>rsTfbae_wUbg_n zjB%8G;sdrW>OtQT*=RrW6m(BLi++<&flTWPsQ>mStKUU}hVzh!-~6P?m0=LSpLuWl z*u*O1A%-NUqe*K8s1}cfYCQVM!Z3#)hKl!6!q0A^BJYdDr6q^@y+48CDOcnzUFB%T>|%C@8Oe1_!b9Gl z!Ax))3MAzik9W~1DFNJCMp3;t7jf`vEAYrLFiCv{_FI;ME;|@K2RxukT8>3uUxC&2 zV_?6n6SZWQL4WZmWG}lck6$LC_Dk;)>EITkyyJ{RTW_J)h%?Y~^b2}4euD82TY%LP zE1&dNB`DmrxvmGFLr-@cO5ZqA(Y$A9AC(Sne?216m6hOms2)5XoCHzKSYo_)INFcU zkS4`v=-W30b-^W|_Ch-TKo#^K*I?0R8L1wUfFk9fT=DD<@Z)b#qscj767>yvMncfN zI}Bc*xeQK^bTHCg8?W7b2b`lS@H=`O`zs!!p4K1){_u7Bt2@PEJI=04vfI9sHuKR*-@#aK3jnUd7t6h-XAD>yaSxZ zb)qz|jB?7CQJrz)pECJQ(5V|ts7VsV0iR<%!p=B(h!J2SjH=p31ZZ1aULgt(e4N%^jzmccV zlUMw_i7CUcL&u9@$ck^gtv}u!#GT858~F`8B9>uiz;nbMA5p?j0k`h^;NF{$9{*d3 zK9$EHxw#YcTpOX?KN(V2Hlb|qITZ9MLDk=iDaIanNK=U;lPZz3ei%pxr=fQD2h^-# z-Up-S;H+H>9`F3Ypv4PZDxA9Bt+hr4Qd*(t^)-zDt(m%{-=?jN1Xq{DgXGm4B2VcBzpW#&sBoI>2a)o?otm=`k&IJA9HkckfVe;RsP2)sp)>y-r2H0e0js13&E- zRP~TYN{6lipREtwZpEm0kw#r|7a(iL9O8aI2{@%ZDy!vT?bisbc_ae0$v9-!^>USa zo>1Y-JLuO1aQa&%G-kzN#Dg$moMH_6g}yqo;D4wX;Y}kQ$}sCo2(q<8f%nToZkz6D5ZxRF;%_&gcAx`v zmp=l&e>e4C^+qIffK?H(03^h1UrL>peIZouN8rut=}No?-VXs7YnLm zYpL6^D3on0LCf>QL8sRoYr|q8;O<`JjV`9GdyU}rkxuB2d4cw0W3as@7FpfDg3Fty zU@lB2BH=;OU7iVtjw=}5(htV&-@)S_De^X!$irecL-B$es1El>EXfCL);)-Jxr-vT zCq%djQSkFPZPfXRW^?P29p_B<(py;1i@-4dV5rWC1ujR9vY&a7k~IKoO)FKtb_Pw~ zamv%YmlsLCQS*{v5LeX!9fy8L!#oS@I@kk6L;?0cGBIxcJ5Z(Eq4kIKK$vrysGj*# zes}`%I!z0Lo9f68wIAZB;oIGn6mOLh@X$Z z`cV&&9eEuB)e*$_$2;IPxN=3;oT0xa5RMv` zFguJIoeM$luYqdZ4#@3l!n(2#7@8>sO;iIIxW9#Bn=pudv~w19nZmP9$ne6g4jfUit$N4!?@2 zcHO{pSOe=CTS4S$M1p?5go1bX@%0N87(~XPSMgPp_smA0hM&k%Prc2`a&=Ig$|9_@ zC*;B{F(~M50O6~h*wxXD<*s+&_3CP5`xT>H)QLT_&w=^AkED|-fO8@l_}61mkljK2 z-u^~5m&HI)*A5&L^b7>`%`}e2pxV)sN`D>#;q@5ev#AO^_Raw1K{fb3V#BWg{bb&+ z(bOq=9CmK6gYL3s=pWDpflmOsY?_a4>r=35>r2c?I*z3?A3#d+U(Ei^C$8*fD0;a7 z-T$Y7iY19?&S*XTdwOWD-U7k%jX?XaKv3yale&s(wE8Rp$(%moF>D)X**`%q%N`WC zTmqMWGGQ?32%0`MNB(^WP`=8hCsr0?ipeu9OPdbSDQ`e@;Uj3~p1{D9j~RbEt3=mk zG}Q5*V*HP2DtND!3mO9n`;P`9j56o4{`Qp{UOY!aMnz&BuL$A|6HvITk@)5+(1-Vv z=o`EQ<6jX^_nimTt1IBkxeloZ-oiAULTni^00yQ1qF+KTjlc!4WJ40F4kgf(R$H5?B0-7I80Kq37O^JRB zn%6JrlIoYJIMl&a-L(e?n|k0?yhXuOB{5!q9TfbT$SQe7)p}jT_m2#CJ^BG8F02E; zps_^rzw=n2F zSo`c5#vfw=&&LO(hI-UxfZ6{aCIBZdidKcJMZYn7>EIC|_>E5|rEYzwSXhbH0S|!} zI35kXDp2%w5gn`#0iT|i$WoAEX?8!FcikXu9bMqpt3y+xAEU<~I#6}L6-5TY#8Bre z^-YQfQT2J~D4l~v`AWvq+$8qdB~X-o8M7q+fzP-7(7#-Tc1gt;;r@~FE9*cMeg`%G zUV`9?Tj(|zL9BN7qPOl|)Ob}A&X^v`KD`F~Ugi>I{V5VZ(g=hG1LXDMe)L#!1_rv* z&~1Dj6--Z}WoI+cb)pJuEnC56EgM~&ZJ~8=Br4hUXkB;_HK)QrHD)?BxR8W__J^RV zcN?e+HF8Pv84~Ef96L-sF%Zg8o=?#?g4w&{yGi}yMkta#r|K11a&>4v*ZNpBj@kMK z6f@P_riqM3JM)e%aX$q$M>@d1p%e6jI-zavNf2ww<*asnYPu{7bDhtFdZICoufGRX zTOVLRX$uM;z2h3y}{w6iCzf=ugmaQ1kT?)a!`a{>)G?2OKPS85gQKD|4u?z))uS^D?&fJAdGugips*Tr1rZlGOhMMM}TG0UDSOMj(*?Q zQQmVE^`66r*v~z9;%YI7Ht!_9%`G4s^91DmRai4jiAx^WW3W<=qQXRoV`YNb%CA_q zwFn|-kA=M@iRho22Lb1pXLsF+DU!dyTG5Xo4_<*s(eK#m`~vmHU4t@V5TgwP8U5Q& z9j<3VU-whk;GPOPPxgSiDiBO`(p zVkU|t`-tRR094(350cUgI4K|o*q@Z7sb&z0Ome7V#})86;|Qs`_o3>+TNKV|<0|)l zA!Sb&g7}dsXy;AB+?}bIW8QA?W;Dlq@=2C)-lfpTpg zvL;5B_`xV52)|Bw$CI$VUjl-<2IBwt1aQQrSl&~Js;%9m%Xl>n`tt`~^L&eO+Y7-m z<19pP_yjI-YU)zejc#2=C}(t+3<4@1Vs*klGEG(6qO-MwC_I`x<-RSaUIF3(w0s*t~v0zCls+u=Z zW2=Wy^s6uO<{u$lw^tIQr#FD}nvM0hHI&n@x;N0Ab$B-ZnJw9jyy5|5w~KnD()t@1+cM;*$t;>+(iAaJ+%G{ z3xzrByv5ECs%a+d=%J!Tp?_hl-t)`idlj!P^TD#vg5l@v$+h5R%jR%H2=C92zI8uwI2o-KSzE<7qpuwAnV;aSS^-=SJ6i>{Zt3y+;V7{dlg$!BOp$8 z8+pCH#9SvDdD}0_g%b@lqjCB1gB$Z7&ylq10w!`!9FQ)2DgF7Km^$P zcapZed{9(c(yT8_AtNFMS*`YjwZ;})^V%2H zZ7p1Lvu9ABn+V3|_v7GWL$sQhjXJA)KrwL`8H`$s8^SMR$_WkXr=O?lOg2?_gyFTy z2=pFLF)L35`YqWg)Rd7p(;-OQcMTM&!zlc$1M`{Qz}fbc(Y?%`c2rZ>8B*YTbP$`a zRp9b&IkLV_ab%T$N4EzxpjRIXhR4rCzfLx2rM_U!-|r|{^b>V2?1GfYCm7aU47Ejf zQ2$UMTKX_&*{=?RwuXY~h8GaH$q`qY)|6wY_!Lh~CNewt`&2aK{{g0F zK4QbfebeiaAy;X=APwh3A0l?zQiFn;&1R}jCekDBNFiu^c#I5fTo`p2%uip|%- zWz~F?CEbV4FJDpc%oM$6W`Isk6{$}-MjV{tkmWI{M3T1@T({U`e9bZF*?$(hMj8=8 zY6FUU)8szNaB%aihRv6%K!3r1BzVk!pwk#j^*fshtAC$kV(>*U>}&+lpc(|;E)ajb zkF;Hk#LmudSQV^fe6ttbJD~;L&uPGAgDyI({eWy4kN8q)f8H*wAbgt>r@e z+k7bFJY{yQ23gOCldA8lkbUYi5o@K%&DXC5?vFK)m6V0b-FLwCw|-E3DBxzamZCH& zh$>XNT;AVPXj6#6ogwId zRe_xo$|2`|9+TNKn)S{Q@!tPGa8osbdR-*p%sx!@PyS02pG#4(GllwuexRqjzF`rk z4m5M>C`+10T2>UHZ1zrIoi{E?&?|!RA0t4uED-e%+yT|TbENCZSK>O^0(dFjL{&YR zOxhI*k$(hW>6TVh3ZD^G>vv$ew#jRCMG(1i1FCoXfWgSgz&d>&+DaQxGfw)NfPzY)TzxN&tN8L6TBgddYfc*EJXNFjl_t=?my1>D zPQd%_#q842G;)suv^FIm`w?<|R_z8KZUP$C?L|q*LuzgM89W-868rRe;O;z!VT0z- zKmG<4Z@NJ}juxZSP%HHJwK8+41Ki9Li2Ap2G;lPNL;FXOC3q3K0^Sl|8H6>hL{;dJruOkj@zJ$|eP*fckpvxQ;_RP^n!Sdfx zXHOi8N*!s>K?lq_)C9b=N~kG)i}JBw(ADw?_At6kZ`cDgfA3cujL@*0GaNbh`uUH|R+m!02TesQ)env}=bzGqeT*kL*X&-50RMD;B2BN`~0ZQV^UefTFkC z!0kgFQIy;v?1FD{_kWLr*fEu;7XBcvhgFbbHwdgZR~(m^)Iq=X86y1U0O6l*A+PIV zp^AM?}nB1iRK;h10*ZLCVY1sIyIigME$2 z!|%`-*$INHa?HK2fkVv+(6yg=|F*0`&m#j^n|2a6zfnR*O)Cml_fo~o25!(F>B#n9 z0)aO~%>QOa^o7=#*b|4_{o>L3=NCq6q;XTn)+1~8S#EsWSBN|I6NJX@v^ZacK9~G4 zjQs)xzUcx_Lsb4)e4Mu`2?&n z*$(>qxYWYG8RQWPj2~DA+E2@n_xvuny-TH?oe`*i&YIa-$>=bK;tbUE`cJZ$8Tmb@mJt;xg9ia1%&&xqYmNgn1T1ATfkzi zyj_1W9@>TH(cegdX6MRLWMPdyk@=K$_zDes@;4;j;-K`G>!j-5NgOjJ4!Yv_l$-X0 z;TzJ4%(Mafr?#QfB}?qzp#X2b7!>2Aa<|(%F`j#tirx29JMjb1aefWzW6Sh z&*4Im>IDqC zhhmWUE|k^uqUc@=sVT?-%i;UL$@#dGVtnpW30c>B8D0N3jp2o^qmJWy6i(>^ zmn$LYQ~v}j%5$(>oDE(~F79V~4Sl@3(9p7g$t>2&E!kNBU+Lv_F&Pu@ z?nmLdJ0$|2(U5ah3?u)RVJNe&>Lp4LJPs$K1EnBtPbGnu{Xq4?msG53#(KAtMEcsC z@FX*sv&obRf$xx;KNWSH+|cwsK-H*N+_3lw=&p?c`^iS&IeIBXGy9*XP;j~J!5E)@ z9ECAOa^u{KsJwX_d~U2pR*im1szECR_r}19^fze#se$oE;c|giG3B@2KpjhS>>t6f zK5&Z4A}hhAR*B{8R%Gc+D-MfpLf-p45~S4$!CDtE>aG|k>6e0yZy|{7q9|+LFyy{q zd_q73?$YiBj~Xiw{pbVX{G&u5&_T%rB~^7wQFrWf+?=9;?UTMi;DvPRcit9ToD0Ek z|78d;9E*m76coLSQTVY|p2%crrD69lUbY6x;?hBynM3Tu8qs49^StUes7Svk?+uY3*H$&BejZnKT z98^UNvt*V_RtFBE`;L0ZIxr8S;%i~(;T5c!?hdS+3v%{^_0-BO1~glr62n;v*p>1L zOgHs_r=A8 z47e`y=Rl;>4p63@XZorD=>B;T`sWML>DOBHsQLhst0#%hnYY9g>`|$zr)6oTSn-zW z%6C+u=Z$sXI%+Glo(f@Ga!Gd0j0$d2z_%M z1wX}d@$b#VZIlKDUzBo5b_{WO-V7s6yBNkK23)$5aOnIXD#{jfl}mlW!AuI~gC6Ml z)f|OY*!j2W7j`doU4j9pb${LW^<%2pV7nFw&LePx} zWc5r(H-oLnK7E7+uJ}kS9i+fMFdCYMjM3&|Ax`3!q2JPNU~l}Ec6T9+|2irHaqv+WvQk$5Mr}ZwO%489Wngo=yB_BBUTD1SY6xlo`pBfnq{{P6( zFKrIWF8%=TlY5}!LMHV1v!MUcB(UC^4q;{?C^P5)7FR>XXWpT(DARH9!vMGzDPT#l z2LXQUBC=BD|eNOxtzPJ0k_!!_R^!C=3(Ae=yzO z4~)MYNyW3msa(e!nkI9=H+mDWwiI!_r%nXUuhTJs(a>5+FQ99N88LG2Lci8@!dktC z)^<997Sq-Byh($V&r`tl;bT;_$0GmqCFVU(p{$l4#nR>4M3scZbzK+o?k%G}r*g^Q z4t>-1+=0tOkhRqD{o}-|vv|Uo0k^FTnUG!$9SE15_2)pySqF zV6E69cgkaUfhN2jZexF0C+XkCg{*}oAeBj|eC<9oJfucZa3w8^U5SFo zk2En!g6i$HG@(KX`rO5&Yf~C(R*A^qlw|PHzeQL+A01mR$D+-TMWCa_Xr=Xqs805z z%rqqBze>?n=PtOiR-x$46YBJyg>H3!5T6z$IO(iH)_V66G0&VTl9tOuR(^!qb3tGq zc#SyS)P~@5JplL>?RUI~L-)#&btjJTKbtT)yb9vJ-bC%lV5nOtL&20^P^dV=wfDG; zzQ$TikG&GsEr>>mT`c;}b%xHjb-*T+@}^W`@%N8Re-%%K4L@mu+fQ^`ONslG7Kna% z9@MOaROI7Rf`%Q z9FE)VvT)?cB#?!^#J%lp7%}Pva^h@&x1~(3nzRO;G)y;e^()gYET9QY|2MQg8tjMN z$BwkwD2!&#O`Qj}zDh^yr{y4fQvq^15p4GOV$V5-Kg_y`5!Q&tt4*0+doYSQyn z3gs*7K$#PQtf_m5a;XpYxx|A0=pE!*VkdM>U^0sVZ4{0y=c+wFpuqm5T)k>CE%Uqz zhDDc2e`zjkpTo=){TAX-7J~+kn@~6}l+h+_SWxi+_Wpc;p%+eqTmCgt{eVKGu`y^$ zUs2z|Wf*yw2jYoblx)?**t9*6c#-)&ADhYB#)?pKC9oU4mT_xZ> zBLi7k@ihKQ4`HPbfsUaenwM^Y&JSgvzWE7L-QIw3;#s2f4MLXdKU|4{35c1wJGizD zm1N_94NJ}!sS?gHpIZiG57 zUemC{1Qh?=pr=_Y>1Al|G__rVb28m`h#i8D`!4mE3eIvwt=nbd>#>4`$!n z&|rlHbeL-)j%QfMn(gv|I)OyF&UmilN8Zm5m zDY%BLK|8zKz>>F>tUOf&vY#Q?bt9VUZIXfExL#s*yaw$LEJb$A6}i3c5x|@8(Z47G zo0cpBRcs)Y8ggju+kL=lnkW|^^~S)?T@e4ze2^Rn!`9KWar6Gx|DPdP8RcUAA6FrD z7~^rOkI7dqF2z_T2j6_-CZf8AVRna6*6c9qR#rrXjkS(`i|51O^xI}n zWG+*|JcMD&$LWDmDL|cn$Af0+c$6+1j#eL-&bxUYu-5M((h6_1m~azPcQ>IrEd_PI zJD~QiFJQQ(oNz9r5#txbL4Ey%T)JcdEz_P1CN)V||Joai9{vSxt<&hCb@f>HOpILy zb1=8~F`pBxGanGq zc`c})_>HjsHvv=alW|(Xb5ws^jM}S>vCsMvI47QgszpX9xhw>@9fQ|KeuhYC21Y8? zP~YAQCjL*s`A-ST12qVLwlVy{4mtZ^hFtvL9uR76p#08El5@TZg#lh%zr(-K=sOuO z?YGa^ADax3e+n>IJRB8CFS+&eZxem%t(3pW9}+{xVQ_vLu)Z%U8Tlq0Sq0|ufVAnD zI^!4?WsXDs)^z~q%aAoiS`u8l60Nt?p=`q!?31f;=n2E>U-?PnN0|cK_>x@bW)gAR z@)GJRRG_bOgz$S(u?pKD@J$R^J--3?1LfFu>IT!Hb0Jly0kYEm0PD;rAbOOHj8bHB z(47#m=@x1Qg@gI&6dWV%f-&MBAbdTS%Gw4%@7!ZlcZ-Pr&U-{vX$gX^MIhZU5u5&E z?wAxNfaw7nWJ?S1P)7=*5&bbDU@kbhenKO!uh^r!h9X55Hl;IObA}%1TYLwfLODG# zUIM}uo;bL62k`cJ$i4g4qh`xHV)OMY$dU|D)v+I0X%`@iGYqBrN6>3s7Z@~dN6WeG z7#92(SgA26A05bOGAmG62Ld~HI0O1b!zY5*lxAFV^Eqh7e+TnrJ-~~PB4Ya(u2)?Ty8LF(a4K6!Ac=#X zvArm;d&kwW9tAduE5I_~1899ahrTC!U{I2Ydp#dPcd3l=tR{Hv*N5o!nc?X4sz{3G zVsMYS57Ig3iJFx|_+=CoLFL3hsvFyw9LFsE2`b+>gZ%tq;OL#CRZD+`Nt!R1RoV+h z@BKl#E}pDxiGaX`=ctzR7pzNW_S>h&w0`dfhM^CJMqTEdCG11FbRih@wSo4DGfeiD z0;)6DXoPJ!?Qmc`!|hD)t9wa2YWu-1hskF9N09%^N+dcT(cp<6=x^~u(}|m4@U0WD z=C5#!TF5r2Oo$q$$vyQM*HZ(o3>^BL_q?*J4{XSiA2ZOD=iI__mNtoB$}aM3=CgCh}p zLNc&l?-%5cyFpTJ4M*-qJ@B)ALSj{aF?~rLji2?EoOq{(4BvhXxMB_Bj#gT~#s;iT zmt#=J2ZpcTM?6Nyp)m9@!-5@09sdeiv@MVLH!^I^crBQQ7tv2BrL5u$3{(+@UN-3% zNlQ@kY&rVrzo4Ox$C*59Bh3OkR504yX;2q>((Zunx@hQEFbq-deuy`C0_(Ims}rQ)bat8v|7K>;&CY!&>7l^mF2WrP{1J%Z}WQfb$!yEbp;w{BY2X~kDjf@9V?sqi(#&jI! z*;KEv8_iw(!9HyTsIJe1tR2%(a^xjZj=4i)rC#X1a}dLJ<)TIy3tl0`xPlAz^w;7h4$;L{=V79ap1#TXwW_J5_)>SZGyB@6583sG=9_QUMO zwI6DsW7K^VZ#hhCQiNdb!u0vwDOi#D5rlES$xWk*p!HfJ^e4X~{aK$t-D$;eK%Y>u z{XVm=*Me-pTQKMyhJ%+sqcuAm(3@dZ7p2PM$~w^SpH9%}DLO3Y+4@#(%*9F;0hxcwDdupkjIpzcqKPnTnLiT45+%0kG$FG+`$cw82$P- z`n}44`n%5HV%r4$Kkh@WK{UoUTqWikZII{d2y}(d*1N?BbXJFHaxj z+a5#bg>rO$UWRsgPasTB7Y#&QSm)D?s_g%vIQ9mJ7hI?8Rqm*_I1XFZ_A#uy7ep_q z0Q=uQP;s*b6lJ!cI-`;*ruxy8^lV(0dK%QPYDw)UFWCG#48>P$(X&|}6#Ap6z&)00 z|86C?mAlfMp)N4?dIg?Scc2Axk4f=xGZt@n54@1oRQ_Qz`246tVc2u1yHW{ir&{jH zCy&wXvo&mZ(}7)07qP?XZ?tDK_lF#rTx5cVM$TWwcz~LP>~S@q2p3?r-wRMq7gF9= z72zL9CjmR_f%V5sblX`<*#`9TOu~K<({o%)PKON55|_=-Swd;$JSwt8R`4 z>GE5k|LQJiRfWQ}T;|T%LR)m1<&199xyZHe1kEw#?nKpFa@wQ@JYMdCb>q%~{zq3( z|C1;0JlFzmzjcG>n>NsAyq=P|eH*XC+ztG-4uXo?QCvTlN}iX2_=W*AwU2<-%Sz;T z{vhg5J2cWTY{Tc@s56ZN&GB)lH=mhT%WS}V97R^SfCQ#|px44r;{hr? z`YG=p4)yi1Ldbc6ffM5>`*%KNg%0B;Fns5s?acjQ?NJz@`ya#`B{EFQNoc?F4(-R> z#H{RGe%lU)Z*>r6JcGe0a^NJK zCA_%ploRMddEWDhU_(FMW$_aOFP*3TOTSQMb|H;5IL7RQZgjtJ0WF=wKzz_24H-^F zntp>MKDCDdk5|~bR1WdWR3vu#NYMC?2T|E3>QbFWc?Cio{5Bp~6Ti{m8+B;ES`N~4 zYryi>uP`vY5qyhpGdcSNbQ!e)qZjr3KZ?%AFXr@n zqz$ddkPKRzw1kkvjwF?2BqYg5l1h>!^|_CcB=j&yl4K+#l|fRe{Lb$m=rz-4KKFgj zxvuMd(BQ!*lx($OCT&S5u^kKZJEJkaM2m@8lqLD=IWucG%*=0BquHxVEb500+~Au4 zi8gYycfG{htpCBFK6&W7@EKq1k%gVot2oj8JXDVQ8v@$iLC0A7-w)l7!RArm*W%q)Gu5XHe_D&`jiJ&vE3={CQ$47Q`w3(2#h`Le z14>>DVO}$iqxWxTAv^35`Nw`{p>Zd;dU-ZetlB|m;u;nc7X_A=O3-B3BDUdqC>%h3K$Qw%y5 z4Q5}q!sO!3iytf&ne z=(7v-HwGA;0EOX~DXqUofO_6(Ui4Qiq{s!DZT=?+o7ZAuy#WNx-3zkij#8)Fg?i)|Ez<$P0OI$NC}D| zIeAu*UYOpu3hNdmV#;pHEp1bScvc4+vn`WmrrVkFX&=(&ZOA`jDqZ&DIY_z~4i!(s zQKBb<&Ox`ZdDdglEve;(*Pejo3`eMRy$#8eZeb<)JTlVyqyDOiC<;BFo0xeX(V9wVp}r!9J~js%}#?^*%>q`n1O3m9jJ3S%4Z%Pjxj%3g1h)NX&i@9 z;nT)^Pe+2ckP52a9|dC@OH^&AeE5bDIRC?8u$i0=CLYEtc<&nw{^>m|RI5>lyabx7 zW0?td(j7J(3>TE6f5@mdjV5?MgES ztGghxxfY7A(>>BZQ_!sxvBgyD_yzC8s&_$!EZpMZ#$iDG7pPZQOMUy%`~2lD7oL}qKfsc5clVg$!daFd zT@Jnx=TUj=B--tJ3CiC+u_*Bf#wzPkv8{?l1$}~yAkw_qpAfqGJs3GtmgkN>RGoN) zMMjh{&$pW*-S&{`)B}S2!AU{X>cEqZ)Pv9U25`1`3$k^KFqHgsmNWk&A7T8IxKKoK z-5OMOPG~~-|7*O9N-}4j{k++1}ngT3;{tu=t4MhFLt09)|bBp4yG*_xb=U!2ev9BkeJN*>alg?ru zFaSkw%BBpSl?ayo2O>QDin3t=g6Q8-Ts84A=1i-??x~sJe7^}iWouCAY{cko+rcWS zC*{=lPjPHNj=_HtCt>do^lBZ86{1o6u2Gn;_Z9DvN7qc03tnc!p&+G`JVrl(Z>0zK@C=3Y zb;qHgzzLT{H9+5EY8d439DNh3!0K%kRD@N5ZgeMC~Xl}U@3_c&mdl!STdSEzKuHD1j&))&zWhN$mss)qr1~;fmulf|8)K_ zWZb{Q%*JbYb6E;#-AY*Kx9i-{Hk}X9q=dd#$xCc`{tV=cCqw;eCDecZ9eu*CgXI)ov|qac z6+O2>B8l6}dLrI4dJh_hlRT^UBaGPd0d)_5LDBE7QjdZHG@LMvI5ClYda)KZ9P7r+ zy+xD}KZNee3*u%Oo!R@@L&4QZ^jX=AkqO^HUFw6bY8$+_@-bR{ ztV4~7JE*I}aPG+(u*^0_(Y5zd1Klfpc=i+h#`ik`r9$QC>rgng6dN!5gQ{pU^CItnVTcE)9bQX)wr#_j z;=`CYE(-(8rKlVtMf;|{82q)Id<)-mMVtSVIt}^_%pP3=!B`JRw!B37=-WckuFI(Q zJHgGRy-<-iNmzZd2}4(XLea@AXk1tT^5@meB-R~bznuX0-U5tFIfmX7k7M*>9~?6# z9j$`edEZf?kbE?OX$IG$wb}?&m&Y)_^clE&))A;}C_t~$Seh>^K*{1JR(&-a-`uN# zwp$vIzp+QD)&$&_+=S%q$A~d>6MPLdn6u#x6#Z{21}FRk<#i;5%5~so_mx@i9EsJJ zrlKp(LRrc~s2PzDlI?HlnTXLdX)#tlrVDFgG!CHt#6IyOm?XxrmhN=0%QXPa`FiFS z=#HY^+o#y?8VXgaT$t;hh9dt^l%L)V3;o7oB%OWLWhbzBj3wyZc?7%vjRxJYA1Pm+ z%AB|6K;K6?G<;23$D(#VaV^ynRX2F`*xfj@w<*YBFlx-7bNOo(_j1c)q0NuD#_%<( zGu{9JQCT?CatWxXRb$-`3exWXV{!W?LC!KAFZO$hT0?zUb|Myy<*T9Ey8siXUMHW8 zOgi+pXQ&-HAF3aGM>9nN8cqr%X3H7GNs%DdM)J@~s&STt3oSR3QL>V;hz0SW_wF;S zJ$nK>zum`UE&EZ{XDq~Loq)RS38?<~Oz_osGS}fNsV5LKN$e`-d)}HEybJ{q+JY#{ z)**&w++IW4cl{?}Y~;G2E7-Y53o(9etbR^9`5Z4xZB1^W)k9O%z4AfHu9uXncmb9N z$3d*@2~>?dhI*3ISkd(!tZv?9?mzl~-6t1N&b-QYyAxmQKp7PKSmMyGR*X2+iuN1a zz;)LtR(t0QcDHEIXZcr%og51ul6_F@I0YPAo`HCT1sWRUGJCLtzK1_U{re{LFW3R~ z%PTj@Mz6``(2f=EjmW`3-V0vIM8hx$?&G<+ZojAz`{qX_3wCT_`G8CLoM1bvx zLabeX6SNMm+00>aP-qj5u3zQMeq=HWJ^uoy>)%JmBNrjr_!XK>TML~Q;V64y!P~Bu zK-NvV|AL#itSAldk;)>ORf|bRL6j@4h7CPyL8g^M^xd8yor;)XdKv1(j$jO9Fze|g z5N{jJa@?MAg{PE7*RBFwlmTm}Ux4~A1>FD5Y4U%_QCs+0$eoi3vKM;=)9)K`WXyZa zwSJFpF6>7O+j`Ks>zFd>6!Uo0fucTzg4@-ntn%GE^naMhilp_RCBJ%WR}i}KS6qK) z53IdX1m3SM0lpHzBQ_t^mqG=@MK@_iR|nBWR3rY?N2T!yF$#C7dOGp^#aKENa)Nl zhtPgqsQSr+sXrt^%%XcJvHZf)W}Bh;=20k~nTeJsX^y4YaozN8D%d(a!_16l=)1~^ z7h92kH7p2b7N5t3RATgIw;>#InMOIW?~*DkrQbO>YbMxT(Rl zB?0Yj%)qS6xnQ^ZC-8i~1AWWy@_M%k5I>538)u5Kb=4^}zgdcxXtHAzU zJdX+|zoM#niaK?YAo1u8i_N>iF`ygOBN{PjZ5`=rC#7=3qb%#b925@sLAX8*jVx8r zOz({T_ERVe*~CnTUjp61N8EQ(GGb9VI88mygO{EJO}`aPf9qk$kOc9}!vZX;>4d_@ zR?M_-#*p>?v0yN;iEsB;=%&-yyb$H6_jP^=#7T*jp%SEkMc^xxH5bKv$Z=4ZeP!UVKLPv9lxO4(;+C+ zwMwPii!jE#nVELp#I#};XdZnTRj&{74Cy(pA2kSyBhI0se>;@o4K#TD9YcT01CMF3 z=&E`H#?ln1yYG*Y!7nknhYtM*D0#H;5g7F49zFZR=+X5Q6IU&w8O0Y!U-Fo8&ezc7 zdIaQDjl;PUBeAgZ8=BMkV%UC}X@1|t<-+eEdPJGUU58n5$<5;x zu+8NRF7thc8r588vGyWZHGN=X1|G$Uiyy+W_(t%ZGKt5pXhY{|9S{&&3*z`tSUsc~ ze5%v1)Rh=NYgePD;{ZlK@Q2n!stYw|Ky>L1Rz@wtn5eTnbKq^#S-wKO?inUqF-(3x z3>9m>=ZT!Ugn}VWQ2FOUZWzZ|2lmv!J#r66!Aob6tv@nLPfHrImbu?EcrVC}c79EJiT@ z&Mh=~7L4M&E2vx7i@!O00ZgiX17mkDNXs~gq1zll`+5S{zedcdcmmb2axhz;4RN#9 z(46Bnm`(Zz%ytxU@&5HtdOinB4_wCN^->i5=sM+1Z7r-#dJO85@3~sFOh_Z$*<_#* zY98+dwN0cT?m8``xTwKM`WZLawn6RM%a9VCh{;0?P=2C<$@`sQx~^L&HquH%TQ2i^ zUN0~yTLa2-Q<#75LGCV|jg|{?QUCQdG;QC8^52qp`4)zXaDCLXqyNz)Kj_X4!F#eu zh#ub$+$*-=+Wz6tlF|nA$?vX>t^@l!G(Y^sSSa7{0$P694vNC9sLDtZ6cZ*(n=8^m zr9KK(aqZZ4RN!XDWnf&slzfmrTr_yz6mbF7THd2Ubl-qk-8szcPDyb?N*;=PFy6?b zK$b?CgsK0N${P?xhuqnx-!m~Izy(a6=`+{cDQx172cU30HpTH2W#=6ya?$F+%u5x- z3VOaq*|9FBIx?E|&FX}}EqAG2xrXX&u{6iDoXJK6O)Ejgi zewfqo1vA~N(6`+dGSYKUFR>6!K3Sos7qJw&lOX%w2at9m9plaTIa zT<6a?(&#%xy%Zopbpz7KyDFPG3o>MeJabqSI0f24^tvNhKgtoUV()NCYELd|z6P>w zSI7gG$_;moq1m7__YLZT*2!PMb<{#8Qz-=RoyTElb1wRvUxsNP$R{=MJBa#}Ni+Y@ zpkenVFdX$S_ztsX17>^yuPyngXlW6g%}zp#;dFFAUqbz3gphkG8w?#Uu=0P6L1j-~ zuYXsf&U_b(>8Az#?HeJvZ43Dq2C=pF1z_cT9mGy=K<>DLsb;O=a$mYHB+t=p$q)?w zt2c-`X642F;g1=lYx@1K8C*)9qVH-WZm}dD>chf$gxy^%aG43xo8i!~;Vf3}e~Qli zV!+nw6V^w}0E?gAK*8&4C zQ~ts4Ha~WRe!D-bC_C{Pi`+7JdHf2r?ytp>dx-mbEsZv(O3B6pdpwsJQ^h#QfeqnE+&iWQ$-_Otz#?Y#1Eci+? zP`R!jX0FtN<$gNLZ*-!_?}>xvegz8^7l9wm2`lY66CeK$E^QCs)9G+5zS@BCrF8zz zf5Ga}7^IgBF~=_l{5^`9vhQJLvityFTXX_k9*hQ=%QZCr#{hjQRzdaF@nELA$2&V( zK`D9wy2;)+@zoueYmE?eH3NhnEYY&hRygLa#DaI>AR4nGuiow;D16U??{X_p{5KT* z_U}Mp$!&BXesAa>#XP=W8EIOks4%S%Ovisl;}svt*KEa<>rYXAU5T3q5(C#U8chvF71R-tsvArwtjaLK$8Oq8M$WD`4hesA)DG%M+@4@N)AG0RT$VsWJ_p!@qB zaPd@u{?=^=|3MK`@IU|zV|@a9?Q@q(Gy&g+`(|lgDN3-Zw=txNKmaC#&+Ln0kQIepeozRbyI_R+HXT3>#99uj;erd z%8PV|{R`4#B4~Wv3ntFH4$&hdm~-Mh%-y^PMO{lUrkp|ewo_Ohwg?vn8-W6r3LdIB zD6Ke!1C-BDGTE9H*{uYL$v`ICnSs@QN$58iKr`etQ#eunC4Dcti4jz<5(L#Z5h$<5 zV=`CpmfuB~^yxNkrf29EG!}~M&Y*ZqFKFxE!FR`Wpz6f~rkRt%yI;i-BkH}N3_Xkm zQKQi6@OY^HNQ}!-wrt_h829a1v-ARS1=@AEJNTXfzSuMb*{JDDJri;2sdLf`I8GuW;Ijx{1@gh?HguaDyMcQkGzp3e-j;-oTYBPr)gheO z|2M2BEprgv(S=LXu{*CBz5Y@`X~1#ZAQr%BQ!>mS5Jf)HB$_`IOMPK7kI5j`!Muy0 z4(`QU7Pdmn)Ht+`OM_!U;n3Vs3A)++_=fUk7$Yfz2)k6AKI9A7jk<>$W_3YCR5mKE z{>y~A2c$79g5dKJsJ$siuOH^1?7wo18C}8s-ixuI^)~8F4MNX=iP-7$jkG=T1Q>J+ znkA!H)`)?i$c&T9slTr!tz_a~mr%XBfoWQP<&K^4SaBd4RC|*szyE}V`W8dwn^Wkj zABOJB9buWZ4#bZRu|Xe52QpB-gU<7Yt9i!fPou!Gt?2nNln0h7OSaj@_dB)$4T)NhZm@6Kjya8%<+s(D?;_91_{7z~YvVEoHZxVV=F zv=4)@ts{c__8-fW);s~vf?1$hW6Fa!R)O)83eZ?inh!KP`}|N) zU0KZwz9nJS=Ve&EC>TYbA5Pi*TO}^s834M(=S=a7Hx~j@u;x15={4(6)N~9|-m1aR zaSgE(JW*sHi1qrV%;6o~)xBaM;KL1Qx$q|xOLX8_U54VoVHhz>1d{*#fUaE!xmCqe zP+3-?pAQ4=xA)9G`%j2#uf@{-iIDt<1|I%fin*P{BVI5K(h`=yL(>RIetd{Ulvtzw z>f;dlxRm+vcvPNF#`vD4*hcrAWL_#Wn;*uPg@;0M@j}qt91jI6A7XJ!A1E4|&eW!- zrH1q0!}Qh*AanEMdZ)^vGLYeltXcBvxRr%9hB04DT9{84Kn|D$1uqjVT8lpAzt4>x< zpcu1=SxtV%%|1%-(?#-Jme-k|G|35N_asa(UyBnv z!|~zX4zvt&p^Vo?$jv+h{>N`~mwy1v1AnD_$X6k>Z#qw2ybW{K6Fc^i0lNGl0{!*9 z&?@{j+D<(OYbzQdrFRvm9@w(Zs~ezUdO1GaSq61aIFMiyNkNgL6(Js-~5ox}sPbnwiGa%JVRGRU>NacJTVbN6_|E#g6@X z5~Jom0M)y-s94k{^<3eE%AKpZy%f2rE(eP`IM?410_CH^P;%XprFapqe19U!k14S> z_7sM4OHjW}7oyJ?fLA<6Kk^DcTy_!VzR#IvL?3SJnt)Y%FF>2C1H=q(fLzayxUi12 z9phf)<^B(|XK2B3MIA)0J3!v}MJOKb#gaRBfyc)e;7k2+r3Go_|5^)fPm7_*@F==o zwBUa8Z#I8= z3ur7Nxhg+S$PoPuTHS13>}(EB_Eh7i3gp!YgPFVkhnxS{2SW=Vf+2Y_O`6QPX4PY+ zUY;g(>2|>2S0ixbf*auZX9hsz6UsN#K!rF9`wpu^;WTLiKZWzoaYrDgdNRtQ@1pvl z0~TC#r=GJf3w?5*8@qHsLHjFgjIw~f(Kj%3Sr)JC_X*8@spaad3w%u70Z>%l6xy=l zScqN@7!Mi-EvJWrXzEH~=MXwsFz)C=Y_(K!vIopwk2S%sj~F6T1IE@`px zbyQsW4hxs}2J7eouA`X*E)%A{3aBGBc^Iq2v-4F#PQXg;2H5A?8wTAEELo}gedI+csl&6vWD z^M>x5n4k=YnvOOYRMC!6!QC)%!$%B!M{J9=Nh~|!A|}j>ft;lM=wJ~Cag)Ad+Vlya z-`GgJ>Hnb7BMt2s1Nm$7Q4A zLB5;%g1`Nk`41MDp}z~=hX}a)FJiDf9?c9@!Qi|;8C5%XqM6NM@}5Xhd?u4yKR*u} z>YH#Pu|0f0KjXqw%A}nBnR|5sln)z&7NG~BDE}1oSIc1J=S=b$6l3-L1gtl>0>#(< z2653O%(#*aMxv7-8LZ(Z`wW=8a~@aNZ6MFhX!QC29{!C5E)F<>GQT`QG~i>dIOG|3 zP4LEU|1W58^ajX_$D_z}g0Q=P4k%wJA;-HN%o8KQsbw)vq`a}hJD=qoXy%nceL>&! zA6Ojy1nUk>K<$>LSUI#8cpAzf^VAg(%yO}M^#BYnK`b0`10+F>;FRHkwk7QlAXkH7 zN;0e8^baN+xC2G6>mX~!JIe1TK%21ua)UDP?k+~n_=8;kQ5Tf=v_rF#pV^q1wUBjt z8E6KrB)xeux;uPEx2bUu97a5_Hwtw9k_+2v>oNV;R&ecp%yc&!xXbTDP`hI?`t8<1 zXy^)DJyMIoq~BH;v`s}AFfyK^ux@g|A_@&x&u zqoCUD#TwRq1nUJaG4tVP%q>x(*i!JzRAW9q-|vbLtz#Krvr>V7A{?0?HaHq1g$B^&|I^C!SEHVj4MTZFbAUeKA@ zO1#o1@Qz2x)aMlVyyz%%>-f~7TG6% zTm>e9Wi0x9BCNJ}20{PFFKBr?lIi& zRuiaG>;=V)<2>z$@!&U%b`bs6DflLT=8_ME;AqhW(qyWe{<4JR#lM5>S42VnKdBq-jx4j+zHppEM{tlPbfG_@jh)zI9|kn#r+1+)X_ zj$mXl+<@xeiq8#{Lk+)8p;=dQV6_}48k8Eyg9gTv6#SPRw3pRqgmCT<&-18ozP zJY(@jaQo#7`kW>YZ=Y9?nExwe_)TUCN74?DY{9-dEeyW*0==zh=g;_)#J>C}XbkagroP#<*{lK&eEkr8EJp6ZX%Z-1d{w*|&G zRe{VP6E>uu2X+1>R<}n$`OP%0I`Rvq=@&!j*xw;KH3_FLr;N6_1k7Co^lWS-uah+~ z3c_&h!i!*Y`xB&uRb%C3B~!L0V242Ww{#F2U>OFDeMwvT?>Oz?X%k!*Q*JhG66#OU zV$xBLmD{?xjiMfc>xxjfioCrJMoe;xGB%sjxz`9+aC4zMkblEQJx|ocZXu=&>6~UE zQ~VBQ67P$+6^$NPe98=sl|Yk7KKoK7oGDW|Um+f(@m|!M}F~w0YWq z)wYK`CVd~z__&-+thT$<~ zz$VK0%udC?IXx*OwNk2m7%apF?}O?540L+_3#uWTuy_TrWbQCQooT|w?f1ca?-%mu zwQx7;RaGCz(_q&{%$-uGAD#eJt|!20*Kut7b0rkD_d}o86EL~+8S%xAF@O8ZP-!NB z_|P3zK1)ZL%rD&d#3`8Rwh4Xdtc|cc1>#@2A?IZqSI^$dRHa?gqT=IV`gATh4d8&E zgRtfWr#(9wZXr!UzX~;U+&m32UOlir=6?|I5;5wpYhd{1EcexWjG^9f&{(norMy3Q zzT1UrmABN582z$X31}35^HxPNm@In7Wd1fnk z&~JRDl8JgX3I+Wi6NC8|rusXF&Pw8VH18J_FTJ>RnI+c5eL-DG9CPcq%s-v3Ax`f? zrnjmPtgd)dzeh8GFv~ny#IN9S^$Lt>zDW72j}UzM0%F8n5dXe{B|l#WQN|?@J-8QW z2CriogZHwmgc-yoDCNcvPeAAGi Fw2~?lHsEbCit111b$b7SQQco4-iG)#Pq%~9 zZ=bkN>Cbp?eIhztJCC~C516UD3HmtuqE**r=G#4&%|G^mn7j+g*G(G9)`iS&({wbs zp<#-GLeN^K^8nHdTvg`8t9yveOU^*){YJd^bsx$K^_i^cXAG((23jmJ6-SvP^rGJN zXfyrB&VZ(QB8yo(1|;vlV?bRQWPdn-4)fY+wx(kl_syBtwB6)ojtAfL)5IDW4Tg%x z{LRH!RL?+ZOwdhcwd@e&EOaI<=SyC^Beu+EwipseZ$+WvA`00LQSkLe!}C2+(oey1EJ8rkUCu*;zhK0U zRQ$B$EP0eCaE<3!9%t|?NdH|4enx+wwW$QkA0?vy8a2o&6kOx#&Gbi}$2RAikbe9Q zd6dtACFLfy*D3dZvy{jEJ_R(IdTv{Og*cV1#4ieE3sc49!5t34Ut*wsSR2|4*~GXU zO3x}01CsV(QIQTz5=!}?^Oe4P%XZ2D3zewWL%|D{VI z`U196_S^B%A&mHsH2e7)Os-Kg<7|7>9Ek?|e9992_<-rUzwl&LGbVUbPx>cm%RjY1 zJzhn8@BxDIvN8T-F_bq;v6V9St`UY9pI(6eehRK`Ie?uBb{KbWALNAO^5FAp!Fi|_ zU0ozRqiO_KX{&|60Af%%Y>~>o+>l07o}v2|?RuFy7_D0;gUK>CsJ`+iR>q_-|EjNS zcWebpvZpc;z0dML%28c%k_)|mK)c*3OfHE9(?o{$Q^~7pw-DTC`aq10iqG8l2ZRo+ zWxm_K;{Y@2^BymU4Ht^h-g_8&t_mSuY&rz`&4IpKBEX=pR`GMd#*&y6^+9TRI7=JIJfOig;7mk3f0) z7_0ht8%)Nff&JianqBlp*UGJwYipdM`tyTOdEyN-E+|1wUNI!za))^0=I8WU2Zgqk zfEoKyld=-ymVP5XaevqllrV8aR0snJXL#$+fF$%c8x2H3Q?n0GpAp3 zxUcE~6P3QpH5sy%Ygl}7L>9b~lo8=&$TgQ?^GI`wBGO*IC=M9P=^t=Bf)Ms2` zH6|VCJK2vb?z|QB-D^PdW+$`+B;eqmi(v4s1e(h~fFij$)_wZ{V{~o2*z9je3qAz$ z=5?6%z#dvwWuT@{77r||!VO2uKs5BOpz1#s^}7$k!$&O?N{%9CjhWCG-Hgfmn_1&o zQw(jMgO(CKXgi+4;(ob+RvRW^)!kxIL z9wGdS3W^WVPP^rMQP%z)V~?bw%6X5lIOQ7V)Y_r%M^{MS_zB&TOrUjW2&|1y#pI1q z=sa^j^{8H`u*(DWlrEw1*cR0DBag)4xlpr)&UiQ4PwKx2l1=lOHj8pTzYjvAxYH0g zOA8{ypea5+mLQrqPDt7w4OzV(gY{lHXu?7IOJx?H8nNie z0cJLQ58D0p9tY4~5&6YS(uwbhMbPzz>n2yB#xxNnkM2+p-o!-V*K#}W>Z4@HP$*q{ z09&d?qhHQ+aJs#c&hUToHt2DT{Vfg^2mTf&o(_lj2M;iEXCsP#_=V=aY0R(P4Jxz7 zLx#^HUhDn_b)Qtsu3nng^q%mU+BpecgU^CE%_o;3-S@Xhn_FSM&>6ff*O# z(fHm+u;^0)fzr)rb*2tYnwn@wnj!bLEXD9Ekr?-CJc!>qQk_pMfTTp;Abtl8DWTx> zM?A~fnF979C0y}Igu1?)xVmenAf07PY@<|gU7~=bQ=Kq!;1}Gm>p3e78Igj**+d%Lo0!W>Z}RyaG^-t$!?e>ysP1XQqze$M zz5P)8zqw4`vl~?A?PziBB=`;*3spio);?;7;wJ&9s4L`Ri)YMjRRm;^mJ+i21r&>Z zM&I1a*nFHa|L4P4W9fAao_7ic%(;!qtr<+d;RBC2uOSvr6o^#i_9EpjY22t^Fk;g< zH2D#UbG`# z^y)buwjEA|nWN6(TG}63b9p}uwLAuK2Xatu7A|PEQSRd1D`q(DEcJ~Cz}~?gBL>l& zYR+u(Kt-WacZaK@9z$c}X7ZYi<;uj?WHnN#dD7BurTcIN4UlcSnT=>OAss1CeBx_HH*quBR^8mrZvAbs%?3erA9#_KPz z%`O_V{``WuG-FLEDMph&$kTUmuux}Wj}10K7<0&mGIhjAaOi;QGh)b^L+tgI-Be@B zxqbw_k8Rh`IfZtq5^phn)^W6c^#SLXP_}i$AUO8%Alm#+dX(}8Gn@MYiY~8&+^BL; zIuxxO*wq{eBUIu`i(dodFb! zhhqH7{lxL5Z0?z-q&o@B{HPjLP9s2A83p-10yyr~fPQ-$R;+IY)$(8%lbVc+we8^i zmE+7g*|bkE05*)ur+Hc|GrSgz-Vu$cSZ57w>)g?CUL?Ad?FADTnu~Aq1U1=B>a6_n zVUG~Vau0?LsaMc#^A%=1Z#mQr8iv*wh>mRJA(DVqi-HK*`%UX&upG#*9kG2 z11J|?UOZ=Fgzp8^$9aKf&;`NqFpL5(<6p(oDD=yaLvMXxei@ zWa{GJIo%hvlZwIq5%KP3j)tVhPzYH=9vb@$^a<=utj%3mT)ZB|>ld)VXCI)d;5cN2 znV_RX2QfU$n7Td_>L%ra=;=iaS$7sn%^#wub+3cTuOCqV>Ks%L+C&~J4aWbp2Sb9% zn`aqFnNv45=r2;oJkpRR~s+h zDIEjan?bB4^EJ47{saDLv;#qCW96i;6kYN}>4JR_dg>lq81W;@@0u|~xq@xp{}vp) zj*@176dUDD;A-#>zqcj>v=4Tnq;)6Pyxzh)B58gW=7TP}$C$S9G0tz(f-3kSC>#27 zX(@T-9bfa+v-V-8>kCwTDwS5$C_($%E$RFt#ErFnhd#3!u(9wXxEUod^?5rs#)a+~ z%2j0UAssg4y%6ga0-IN6fa*e*(Do`9wI`^=J#Wy_Z^)6O9)oYx{ZFn(42kwQ1&S|6&9zb;{0ix ze9gf;aoj^F<`$^z(*?mJ%4w(C9x#0I0$SJC;M~}YkQUZ}t1o{>u|0h=(| zJsZknLW!a3gW8x6V7uck2#v3?tW~W5?6c*$uWn+?K!1=V zUV!3b|G_711X!LXpUUNtkVtn≻{>R=8v0zW*ryMf~7pTcGvEC1@#o35`@^WHnQt z{!=RWQHH7fgg1ol`OHE_63b&zEgv-KHu(CC1<9Q4=-3sEGw)o*`jRuOe$Q>Ro3e$x z470e+tRy;Hs?fmn0Orrm#_k=ZD6tMh-RD@a$`-gu811xuP$*bW*hX4+6GSu*MU{;9 zn>F0#uD#!}zU&hg^3$Z1nPK+z1L!y|1#R7qg3mNlw6>wztLY2#y7v}x(!(Ju?hfQi zh}pWhmfOYaqiFg|u$xC5?Ub2lvGXog+usH26M+7&wYZyq1!=Py)NQ+2WuOr%-b6ug z+Gq6QoMoLfME}kAxpwG45S=;*`V;K2{KtPlkzqR^=Pw^nA}Y< z%y@kms;66%U!fiAPW}ILISQJG`&fwGbK2RN3(8H~vHVpPihE}W1=ENh{B1Gn7H(jX z zZ>Gr1hT_7v*D06!2j%+O(O%Zf(l+JR|+l z`n&^(%#)?*%2&k5_$-KW@4k7Jyaxxk`|f#=oOA*6rL_M}MqVbq2ph+or}~jJ zzuvo<>RcMC?;PY=(=I}5_Ziwn`h+)}cnuNC)!=o^8dRHF1<6V)^sya@DrG8co{>iW zW7_pnHX1bNw3MqWB){WQXnZt*>fXUpPxUv*u-HO1R}E92J}xz%xF5vfwNjTG)a&i| z2igwOPMCmruDxc7TB?6tiuywB)C*XZU4Z74*O31+nN?58#JJi?=wq*ko>`Y^X7CyV zhaU&SLAe;-BN;r6pFuO-xjrUCz-!ea$^+}M+%uG$-}jJs)sE0`i@uYYi`3k!3e1O+ z-aNJnEPtk*=reC&=TK8{3GazP1^eiEUgB}GWvH@F6psCP0=*x0fuX7!wSmX@GRIVW zv#tPD_4-&)763w!3sl?iUhWkT}V{mf3g9YUALV0Ua7 zH1x;>zkQc*=1kg=JA5!{m;s=C7l)r#cVOq_MDitW2bF#@RC*@zHsgPB+3}MQ-0V-e zhc(c#_&H9z*Mz&%a#1L8glP3TXwh7Sn0Zaye^(=P77hce!kf%8whKJEIZQXzQf~ba z`A$Db1?^e1o|%EWSJqQT>@3&i8#9xuXSnO?v7neZmH!_{XB-e?`nBPa5Mq-MlG(N; z2_bDX&uQ4SNeC^W%^+M#eO;oI4Iqe`p~&85jnYXng*Z2rY3BAq-T!oz^qM5R88Ma{f}~m#l(&1N?rcS*JV&01g{*GD zEew9XpBx&cOziVRVY!6PAa-9BE!&Dvz1A3WsVn9G_g(P3x(&`REZR|iq~G?ul@lVLbyV6Oc~EPgu%!YY*Hk64FQtsfw< zS1#1uzYiPtc9An96vO|`2l+SZx~?zgmF2F)%PB(TrIm_=cM8<#w-IaPHuK$b1Y6@@ z(R@NrR6KJr_|-Zy?StX??fy4Ry&OezyzwkM-4)BuEl2ajQ`F1ciK@YCP_wEKVkmzc zKs$3+^HUHxY&Hhm8vvdnchnpoL#|+o%hq;abIT?u|6?wijWz_;+#)Rd{Sjnsya3|i z){5q#IV`l_DazTzvd-h=1KB?U9KF|(zcLK;+8UYFtfk<1ashM%{6TZH1z5FsFSf-} zpSG3maNd-anWe&6#>6DiW!ItBcb<5|T zqipFJAce5M_TUuRDG2*;0@eN@+_~;IPlGdQj}b9(M#kZm!YQnAWm6EtP3Z76khZ~ zZ|~VSWyfyRQLVtJvHL(OFj6%8-{4J2>DVG7Z$|7aMa7XrXi2UZ@%%4rxzT+TY>wx0 zk2sMYd9Ku>Ou6HugM9UR`iwemiZTl~qA-p4A9gQTLF!4`bBu!ML66W_DuaNKzF3_0 znyELNqtSD6PXEkAtJ`k0qp4PWbAOCkHJ{O9!$33|y9QzoJR>(}1h|y-;~jlogJ17x z4wBor=owb-JwoH@4a-lfBS`}pFV+g`O|Q@$7fhh-A0ew z`OsEN-I}Kv#PAhk$7W|pnDZZ~yS9Sx{7h)q5IffYzOAMGb{;ioC69XP0pjwhtohbC z6hEATvJ2DDpq8>$j@b$q+ex_d#94C0#Xv*88Zx&}WfpzTVGcddn*TM25@kFJ3X??c zyL!^!H_f3}C1cF@5n%i)-9wff$0)6f%tK5)1jC7#ryyl?yqmh(X7)!ZcC5k6sfs)=Dv@VV#PV6>R zZa66_pBl!p{R}8y?8<^JM6xJR5xP7Mqzqatlp8Nb&wWmqyU7VG={H}rt^hiE2BPXk z6*p_RiCJ$>;rYim(f;;-XyBMf-pYJpU%tR#>xVee{TAr?TVv6#WDrQc*@&or7vBCE z6_0eFqvbJre>lNXYCm!Dq9(+7-5|Y3pWT9aB6aI>u6b||eC_>kbx-PG^*)3B^s^x# zK8$h|by)x88rryJaq+}IM9oK5g5Q1@t{zmurQIo@xo(W*#?(_9_6FoyMwtA@05;BQ zfb?hu`b|0oZAa_DIp-9Xm;c9=pR=j!U(KUz-O0((11%>sg2$O0R4t##S`N2CUH)4% zrq3%pwH6HCnS*cT6Z$PXLcyaep#A<17*Fp+Wm7jM*ic{Fc?w!KB53?hV?W(*&^`GF zI^;b=iE=T|`6UxlUN^C}OJ!Ivtr+CH_CW5L1(;jZ%(NYxD4RNtW--ZFZX3=@bYnm` zO27qO^LgCS-Mp-44F*OaRC=$3%vBG0is5v~qh9Kie)nMK@mF-#I>QCub!{UD(b;L9 z7A7T;&&K&ATK7vu!TH>;;BMzMIq~lP-J6L2Ak6u4dV#|vLP|&?T?#dLDMpDKnDoNy0u1kGP>h66{VA9mR z(0J|_Oc{?L*dL2-FWzzKkEbGq@hfaTlE!o#vav9jIFB1-1SFnGmmrP<-YOwEQ)lMZV3(THnKGlj&JY#!-Je1$cL_$1cn1kT z_F~z<3~0;$1~uM0!D;Se)E@Vpx(!XNyPI++9{ZU(zYpz9j_1XM4#kPxsgU&=p-$^E zdJohBx99X(rH$gMZg-GhKhHBiNuXh0Hu`DKGQFv{dDMqu@Xk5TLpo?zR<#x!jr~#T zm8`I=-oiW#sv+bT>Tx$NQaFPIbb363!24}r({Cymp8E|Q%=be3^GFo0Ss;>H9mK5P zD{M30>jHcm}br$*5#Ur4^CBslJW)V6ZgT41^Xdt(PggJqW~OO0r-Ve)+lft zhV1r)$P=Yty^bitC`!bNUn22A05x7A>MtC@}CDq181ax;BK6) zRH)`IPFJ|`@Izold#034>0J5rsAy3non`iiGs&#IXyI~}x<1}a`|$(l>h=ds@O##{ zD;s1UH^Ft+7;MrcLfedJObA^}?A*yXYf&+jyxxbcyF0Pb!j#gP4>Ttv-h?F%}qbJ(&6{T9EyO@<(L@u;1N>Fe{0AaHrZZWtH#8_Rpj~ww8W`LoZa^t29R`ZTCuvqW{3S{p(nNvLsTBL^9RN##9e3jl+vY}stmrUTPpCt4`@LZJZWd%ty91V~ z=g~Uw9yY7vS-GVH%Nv=1;vRiO?J7sOH4V0bUJ65QTN>t;+5^- z5^{x0O-k|&J2JT9#bXd0mP78`D_lA6y&~v+4HK?cLkh8k1=W4?S9jK+x%+WQ5yvxw zs{L5yS_{^vq!2=x#k#?dK;>JYkdA(&$Q)1t>Lj|;SYPMKT|&%>4g*Q(-HR3Rg5+W4hUQ>=?5JZ6>#~6n+l0Gvc6N z8?onFdoj(04bV27=INy zt}JTw5AKrq5REm}Xr!eN=j|vvEHV-m_actY&Tg2|{}%E8_MoWv3Hjf)fZ)Stkzv?k z$~d+_ZOsd4?fD1_qvcpPYA5P9bikO-H)v!_?;I$_fHyMQWqB!v+C@Q4uq6}@>;Ul* z@(Cqhzl&f#vzf z+-R0LI zp8h*%EGkBm8wf$0{XllQnRmVZN;zvkNUEpK(T*z0$F+%K|9pYIe;fw!4+YZ=wnxXW zBT;ZEQ6cM5$RtJ)SUxucmA>;?biZ*6JQu8o3>(@b94F7hw9g>e_}2Ep_&wmK z^nv-O>C9`<&0>yCB)8BGX0K}pqCEq^;-V3zM5tlslV<93>7lV(7Wl<|W# z7|Skm!OL2aY;_Zc`W8Uux#LWtEho3)O!yM3-!K=nsL8CV|^KB zt{6)h7zGnHowmf`!J!;PGf|?`UT(EwH zty@ANFIfH(d&^!!_DU6Qtf$@2^cLGP(Mgb<+0BaUC!l)i8D8W^-{F;hJmcYUkpHBd z{hXe*X783E7VZY|@1GTd;z=~m8w!!f7D13O2qoGok#rZGa|Tl1b1k`m7tBDNz#x?F z%iv*yQ=wz{uZU6mQ6cbws_JBvRsRVy&b)`tIZxrx`D!$N*?_rDJ3#3+jAocM7$DyX z)(#J$uILudww|@b(NxM;#O}#?v zlZxEH<)HANk4xO{U|G>DjG8evHfq5dmQ~q)V<;P6yXx#wXZpX2VI31opE&}|S0+j`QAUAb8KDbf{ z5!oNnD~vkzww?Jk(w=m__#GT~y~F_h%Yec=7}lHiySc-l_ zXd2TQ^(WeI-GjMV7kHRxE~bo_iIy=EUinZDqxK5HdC+W_l1t~dw!2(*)DGRHv*Fv6 z8)#s+1iOy^3(Kcv;?UuAK+ivq5yQjKOj*iOyXkqQ+z*z!445j#f-g7BCBJPb1l(!^ znOiFFsuH5^%B|Et%f_JH$GO!TW01Dr<Z=W-j$&Rjs{e{Le* z!xM`<1#JoM}q*&?AN%KKG`e z#rh!74xsM!YC4O*Gv-MiXHh7UiM&)X^o(txOl~xfznTG!-2rG+MO~aSnp2$`#j7Zz z>=N4pHCENk?dc}e@z{jYv$ibdRVpO7*kQ9FF--pmqioqph4{3oNNhPrZY( zb>9#RaE<{~mi>*n1IL2uWG~`+Uxus*>U0pNC}h!L`aC)?rPK;qnhKz6*(H!vm5cn+ zMW|O1j(I-!L2lx~9i4`u^NYjaH)1w=r{}QDcg+~%5QJkAi_!J?a5M=!0X6o+VN7== z`o*8d2J;)JoSn|YLR!Fb+a5?bOwXT}r?}hI30Ssu3ix>)XG4WyAn!@<*@W{D7S@4v zJsv=M@|d(V|9Sw;N(*Kycl`2}kF(cj`)E%4q& z`R6)EY&ww-<=@tzU&0*_OpX(|+ZW{~T;v*ShLxXgLZ*!cG|!KLa^Hts z){|q%ox|i<-9z3*eUany=jbwG2)DXwNsf(G`C_GsNZ>F|ZrmdfONmIPaR#9D5O~azoKPvjP%`4`Uhrksl(jmhRJfjOzcLSseTh zDlAb5c5Fd~=Q!fgg+m-Y+oVJH@a`w+7`*lpv4^LlyyrnhuFeUtIUmX8IW17sU4f$Y zKQOe%8E~wpGk2OXx_yv=aNBFH{;h=zu2X0E8BbTUXZTbTI6eAe}G5@hS~vHA1jxG@kE)+X6o&3g(0=w0&?}t?a<_QmK}VqO^f6GTTmeF@ zMD(0H5;S*SQI}#b+vv0pT{YHVs!Kbt!IeBH^Anx(_JiUtBXEmc0_pQsqtvzm$L!yY z75pM}e|Z5eKRYmcf_*qKRPf?nO&oY2JlpHs$cww6fw zbR%>7{tVB5y9ldOzkuI^r`%Le2a1-S1N)s>#6mp)6_Qqn>b9ZIL>y!gJ2F<51wMDF z<2pAD)gfD%-$2e}D_(Hbs@ID2zy|17N&N9&wNYn!9gA^d@POP5zNc~qRTxShbqeP{P$YoLu}&0kyc#qSu-rkAtsTB z0Ca{w0Y&FYJhbf!*!vHM=2Ir9dz^ZceLq8^!8jE9goAGR08B|OA_nwRoY9s6@`EE; zg7*qcjph$EFYuyvTfp|l4)dDO& z-5(s~pTYVu?MD7-W6jskVfOb#@(|czP+2r_ud{C0PxygNai=L8@D@$4>XLt22nI%v zFk-a~I&W1$uKo@TSyl;6(e?D~=*tT4en)ZLX_4aPVAOc-2k(7XnOL!xNA{SB!hhz0 zW_1e>qj#~O?@kcLjASvf!(fKj36vfhr?6}sLi}rQrgJ<68e?A*qjeyYy!{EX-9H(dk*lZ9Rc;4PS_r)f2mlJh9nq9F916 z8p8Ul2i;zCF`It#T0ahBbE74C3e9oi3k}L{@1ZV32A6)d6y;i*fI<06?CRx=<<(DM zN^d%!Srx+kNry1#+z^o79EV2J|3)2U0&6_>6$H;#*#_ySQm=g=o3XM1Y>ZZecuYJt zPdmu;?_H(6o0tpNeH3}uPNnnkAdFl1ocD8m2FmbYbkn6g<&8^d_3%5izNPu^g_&6Y zt`4Pp>J)C)#XOR_y#@XELgw1%T={1?6GR>(zJmpmZuwPYarPe^@$VOOUR+JNzI+hG zMq=Q;izpc_VdeEhd7}Oal$XbfT%4MD%$_sgK4u~mdJzN8Z58D9dB6+*vjKa0{%1ty zf*|)(egSI*dB=TLH{*B6ObFwi-G?x|F&*<>p2b1nH3ZuNjyX#4$}V@ilF;SZi_}IQKq@vBX7`c<-k!S2Id)pU$_O zbdg6UoTDs80?I#mqNU@1Aa>j$stO82MRX4^)VYKmTYH1=ivbw6Sqv6CZHVOwqSP5V zxFo6)H2Fh`p(5a?>RGuR{l*uqj|V|wtZnL?H>e!uh2K)r zam48RC|vegQ5G{8V{Hmi^Z7e^f0r_WMYe79jI)?=_X)U-@doG6-dGds1fG*mgQW>E z$72w}{4Rl5z`=Rcudwqhhm3}3aPb!spCAoK{PP8JM?Apg8;_vmVHJ-`b7AVCb=+q3 zW!CXi3mx~z@gT2ih}EVU=f^Y9Xu1Pp>OD~Ni;}xvnSs()scr4jYcw10i|UCQ8PB@OS5BFLTUUuu z*f&4DayFW#g+cN0T;#IvA8?+8w5e*v-+ufT8AUI^&6heG}p&Gu`U_31*Cgr82`7|v2&9OPY(iT4r}00yf}utom} zh$D@7Qbr1@=B$UTX=gC#&krnU(lM5N*B_F13Fw|?2g1*vaHyREb@$ytGc|ozJr|=q z_@F}c`&7_1Zh<-@eU#`OQy5m>f{pG2b#*taMKSaNOnn5MW z5xF@8p!@C@5TAS*O(+j0xM|GAKkFc6p%JE!*G6waH7?(?4|4ySPCTLOknw#FY7gFq z;xIp^=dqk=FYb%=iRIwDI~mNEyg;i(vDi9Rg#)jifpn|G5IOoL6kaUA^xH?sk5Yl2 z=e|Ige>*fkyn%+p-f**wgIxCYKgfPI8OvwiAa3kbQAh4zn6E_+;f#|YaJZDOAG(iN zH}^p{w+8)UvLScTQ$Fi@7Zl#0JfZ6#9ID$1501A$!Z8=LP%9y}tOkXlyLsYj%6d#b z1cDu}q19qKad_4!+$@F@Z{je@v*&{=C~qLm^dKi84dqqh2lLvhL6f#;-wf z=!@!f3{D~j{yJtvhZyDurzA!`L-)Bg#}4 z(LHh*nEgiH;eSu_L!SGZv5-1S zvt2-NU@{B&Ifpz^Izo8p&N~lwxjli{%E$N4eN5o(XBv*51Njnc*aWV zu-w6K(MjO_vjr;ePX@i=&$#Dn1-e8ZWTM-yDDYfK&#)V;;_5ldYQ2Tdhje}${2l^= zov}#L2CetXp0b1<<8I|?x`GR)pUZ1cQUjd+XF*=ig2ReYs~&V3gtoHxVRN~ zbF~6ubIP$e=Wi_f^&^I!YQS$#h^eI}M@vl+)(oJG_QP(p8QvR()AozvKhZo}@fuw= z%DLdf9Fb;yDq0y8LQ%gKbQs+cL!*IH0ip9vC|o!DCK2E7|fFgfZx z`1-Y@>%9daof;wvSH{p@$O?jbUIU}nLQuK3@w#VgF{f`1&2w9^aJ?^e^1RU5#0$iy zE-G9S9&mx6bViH)S8#5knMKln=rZe1=9dt}4X9hIc(?^DI^uA$ z(HhLX-^nfhlwi%a4sM>*3cAChp)!zi)cY)W_NghLcbxpKA*np;*KcT9sYc1DXjaoj zeis`h29EB;LOXq!HTf6{?iS|<+<1o3v+jbs<#SX|b^)bBI&(|8#f*pi#AKVFP!~n- z?j2Y1XI*Ln!Oo$!mX0((p8Y$P|D4U7hcrXwWf|(64h5ytO>jBc#xf@i=P6}(ptmD+ zc5W@m?>G^IDxc#}q_~OZ-(O+a4gm`Hs#&1yD>P3`) zu<>4Rbc|d}Ug#>8;Sz^#E68QPdoG4vyooK8JushNPb?>I@a%O53%>0`!=mAkIV=pV z9!PPC;S+2;w*Y)4$FO4Ib0`=6#T|QlfM~~ZjM%KihB?t7`Z5YrwrHSx&3VYY)eCCe z%6Lp$7&;oIvi7!0NF}a+xKS(kCEN1u?FE$cO37| z9-StC7u_ops`ENd&H&3k&LFt)86}cg&^?Iwa<@P8fMF}ZXjmrNAJV5i%5yYtE~0ae zIfKimLHt)=SX4-EveYk_b&(h$<9`Ri+XhkD=61A6`oPYAh=bzDlMuJPi1j;Cj9D~S zc5uoEi#7-J#*eJ-SrfL;r@inHV!4=KAg@rd!t?!Yuo~S9bUZ0HIdT|Sef0phxLi~^ zZD1KADp1y4%!75qxqCQS>$6l#X_3@ zL%prz$d7AIJoMkuFsp?bZYMsvNg?-4JPXPmExezO6xyW+V3zK8%w7%Mug zM;fLRt#aA)#e2wFuupw`joSYiT__w)`pv`-W{hnP>t zZAIP=9a!Uafi1aBF1EW?=-E&Ml8LFkWYG z*r8}Jy8zwy_kgax2-tKdJ8my z-@#~(Cq}(m&$8RgvF5+e#7Bw3z)e>m(umFniGT6)b7HB>+kzIo7J<>;Q0(4L9EA7* zSo~u%IR1=bmDh;0+Lv2kZ1hD8k=laQkfW$~>^)?8KS86_4?uq5p2+BK zKQMj%5e)y?%T3+JLRaf2bR8(8{J{#STNr^+bCMwQs1=>LY5q^Hce(3KUh`=N#`PNu z3O(v64%|dc`W6h})K|}NKxZ>+bhp=r@Y3%Xo?U|umjC0U8KGbFCB8E!2}^z+A{N;| zNPhE*?wKc;*M1H;z22cEkA#gQ!!g|T1VnG!4@NPRcXF7-y>}_0rZkT!(j35G<{&WK z9RB|u{*=6v81LBuDPQ*!>w$Ka!fCd$WBXW(EQS~)pKUYso49@KYAp0V39GF*rmTAj zVUe!*ZC*B-SigmGFUmW7zKKO_A4K+~^UpnhkmRIr%WXaR60>~N>^8tW%D9;{&^yRM zV=JhCnx}g63kd%_&8+?$gQf@m#LhoE(N{19qc(ryl8;(k@W&X1XoNWgq>=O4^F8X0 zPeeWA3DEljaUiU#(RrO0d3kB4^2=mYzFQ*-IWQ3&O~bHwSq#&@&=+^cRzgPXX)wDy z1horJVWL+p)HI)GqQV7O*>4sIieBf5tfMKfmdG?;TfqGYbsz`*MV-wH9Iiw|&WMvJ z-nJLDwI@+8r-s!iHZcZiMqf|kEuz%1u?5IXG%`Z>07^C=n7 zlzxQRc_~b1(H~HGW+Ixdro3FBCQrNLGlKFOq|BWUO{ZVbGjk4x7w$&+f}N~Grxu+C z5bIiXkQoL14TTlwFnYmJ5cR0W=;(*&=5`V7S4;vupT20GL7tU$OPI@>8$8qa7pN;; zfpIlYc-5O*U~!&UJDq=%hifO-{!ab36?qDmuucppDg;4TCNFlXW-0?MtXZxH#$_j{ zH{1_ZudaYk{X@(hz5_G^EC2?`Dj2Ic)`&oaB(S=J$Vma zZ{C3WG~!T60`>k@Wn1TgTUoGWdfb=F~4ZX7(xAN%Pc_?GGU8Qg7}O z_mPWC-+=Gp<=`BZ0ICD`q4mybD4uHvnLCILx$KLgV~;huUAfM+jj|xeu^TmBw=s9b zWiE?60RgUF(A=w#3Gzy36#qtD+;7%6#_}!~th2&zK zmjaS!KNY6Mspw;H7h!HZr2KrttbaR<1!KcNqH_z|Tsy$;M=fht++L0<=%U2e!K;As*{-df)DL!7NDW&PH;K#k;yH$5*K5Ns4jOalo;NCZnHM> zYV}im^Q^+i8<)ZJiWX)*`^iR3-w#z6_mbl~2`twf#K_E7P&dj3bE8i2)E43(F7ZWo z@o(t;-XBVK>_&%AcR}nl8_ee2Wy<60Sny_=n~8I{y1F;0oSZS>&}vlI{=#)W=V7z^ zd>DHCF$S#2K&f~=b93I!ZM+lF%5E{ZjH*ZVjN$8WQ9{Xrhak=k<^ePRfe|q; zap=n3V6rzIix-EZ-+cPp+h%85|C+ci&%co4pqME;&vKpj@vO|M9Rv+G^9v_XFY06f z>zH7Gt9M_(^20hz9I=HZL@q`DDVI?d@4!S0aZ$Bu+*ZPT6l)Dk!-efDC8ELCLno9fekvwUt2HZw};kC04gJI})STyt{$m^q6 zZre-p*x$8v(HYGW`qOh$?FoffL%^`%J1gGg299oTu7xJ;aJ+=*gg_1fpBORH)8pHG1D70kQwZ| z3tetv>^I;9%-?$$9OF5DYgU7_dIsgewLuVb*4DV}BB(b3_nx^O#m#?+6wChwy@pV3 zpJ$94St?k4-vy~p71(Gu8-ntjDLYoCSmH)!+2bo^FU|=uX+y$Mu@f6LiZ(qei)kLhBJs2~hkAZ7ZB4{j1zbq6T-G&x|}?nD|>@rQEvzueFR*Z`Y>tbo&2u4VldqCE7atrGJ}aK^ji~#MpIXT z^G}9#A8BWoI$Ggke~$I5Ay3-}avQA+g~o6l%H_mRub_ndr=hlC(JwLk4<9hHv&PKC zLLTj{hQMbhV8-SgOqp^Tq883mv zilDPf4{}VdpnFRZWwhKObRlu^4^sx)I~C&&=%dSt!I1uPE(*p^hb);AwVj`V)6TC@ z96AfLwI4xcjU}2KD*(-v7uX)uMR&qd-ep7G<1>%J&vPgUu3duo)V92iv6g^z&Y>!>EbZW_=KDbKXedZcnwZVWx3st0#lj48CqvBShk2a|?J=KU162`u|E5#+zBGYpuTY{_+$)rg zDn$4D?WkTYhRhztJnU;5N@R&_V!=zabQ;f!KObg7p$SB-ImC(-QIw6HiJB1dlev7J zF7f=VkO#d$ixJadqw)^;PW}K!KZq56woNgm?-dlJrK9{xPbkk?h2?gZtft{3dEoCd zOW4LeiH{?94CN^U2BAFkIP3Nye#F8>`61Nr3s_-=M*Wte<(otcR-d^Cf>sW7o%htc_y<(A z4PZt*y(R1RVv0^W)6pS!nxqxWr~k_An?FHu+ivXsC`Gx}IW#&u4CASnpRkDfJW)c> z5jTO(>;TApU&~XrKfxsj^C35IDrK$Sf%=*eNE%&GMJ&azeST29ikwUz$icgHH5Pw9 z2ys7tu!fdqh&dJradSq1acUmyJQoUu!HXc|`ggF5n*}bLGr>(s4r9NbAZ!U2iB}|w ztb4R!^27_6ymURZIn(Ur`Vh*cDj=O^${uI0{4XPj>9!sy{luC5nLLQ@JOaIo3eoI> zoHzy46S_JAl}{{Ldvr2p>`jNTt-0h090w2j%h2VVgvl#6fOPk-BJqruA{EWYq_Y!j z&*ybuIc1Jhrut&*As=$Uf57y^gFvBb2j`a?P<;ClYh1Gf^)xaT;64n5b`wF=llBpB z*0RVm!KmGI83cRkZT0k3-16VSm@One#nB-sDc_?|zaB+wqNBLv0kKi~zrtXxC}M?I zpk@ELJgS{?4le?k-=e)3NblUr=;a9g=uZ2lHwu={z%JW*Ff2I^Zc{0@cPEV-?np$V zuNj~&d&~o(93ao@9r-I_c+uZSQKzesIo{p{B`dngS>G2@4;(?m%1bmSRD#^tncmx{ znV>mxdh>+y7`A8{#0YhtkVvQP`E5gvQ!H;HrIkpmM5DzM+U2f}l9=M02pZ=d% zbfF$4#=mgAx4+?x^W-8sJqXn|$HAdiIhcJeh0NqMklO5mu7X}5tnlJaLh60qtwlYH znb>>UG3fW?B{W$SVAY&rVt$+y1ud@N@+@6E?|T!JRec~UvH}{P`~`Z^eW2d>I_l)k z#;_D`uxzV<;`Zxc6Hi{;rZTQRvzECfYKWII7|N3v_dAfpJ*SWpZtHUNvyb4;G4y+U zvz2vJT!X~D*&wN0joq8>qSg1Upn2X03QHT%V0j>VI5wet+(V}BKZ@nVK1ZeNS5e_( zHKvFsgQ~}Eh2X++Mdn{ex%l*V=3&}`j%Qb*a{K{S2?IdTmNrB6cQ}?G$!3{nmZET; zBQNe+jAFe+VmP&mES9=~URE&FZ)(TvzsxXsOAU@OiwFJeFi=0wLyL43<%3Tt9$00N z8`jI#-c28MFC!`+P6gMX2iVc{63bS3LD*G$^!)fAScSjE!kwHtuMkv*SK68;Z=3-m_);OCTYFUZ)3WZmea?;`n_gbc|7SIMaD4FcJ^lF2Y$+ zRag>BzQMkhC|PC?If1!o|NS@en&uLFGTb(0ND_{by#nJuj-YViDsH%&IAD8d2N)z` z%8b6yyI>C{J>QF#O_WLc{1lbOsS0u_kO$;%@ZA$iZeYs6$#=34+WYDRtOC&)+A7iC z;Gw_r!AN^IsL!pW3=;ib&4%)Xh(TymOgSXYXoXknMRF%~fth0>i#b;a+WC(#_^+oB zQ0WQ!79S~NLY$6wN+!^soo_Tt1?OpRlpRBJhWi@sSpR_Xg(Xbo_Kgcy3;9N0;)q_#;v484bp|}o)}i7f-FHIjn0qY(G=7BD>C~6~ z7|7a|{UqP}NAP}F1>HBvj~h6G%U_*Q;D1L@_IodI`S%wtbXbXj?QxXdTdjy+nnrA> za8dS{^O*D_68%(X8JtOk4xtDfrF0MedWmOIKWjwW2W-iI2q7ox@f&5oy@%)V{ zi$~P&%Mv+G>w{+Vzj689zj(rxd1&*IbN?y@F^+C9p>wff%FP;V@+rai3i8>E{>+n} zmV@8Q=`3Kw9w^_c%fu{Fq4JGlmXD<9y^(woW&_c3b`nk*wioQ(-LdLfIha-4gYXO2 z!Qyc+IV;7iDVy#XmVZD^`6=!<_9I_Jb8$h44a5bhA>@((MFUTR?(zkY7nz7<)u&Ns zgb_;ST*A!U{*bJGhrPFELWOz{?bs)y;Fq1y9lskJ4@fX*of|pzCW*vxzbe9?{{SKJ zWip@dX9fWlsM38*te{h9IBqU+zou}(_7a%*Zw<(A?m&BWF?v1#nnhmV*{=Cu_Og*F z3pTKnI|q63k73vmu^u2&4bJmDQOSlXgqCm7Xmd44CXePZk2=9e>j=WWGAP>2F~PJJ zM_8tUq~U-_8aM<;^pv4oCluAXDXc{M1+kLEDA0_XVZ0(5B$G^WXml~y_@b%C}dreS5@7g%~955a11&%xCkyBHF!d*X^|0d#K`_Z${oc>M?H$b5L zBP`i^1*E^}F=gun9?_HuzW%{rn3lp*>}P^p?uL4S67+O6fia`@phc-J?MX^O9k`k$ zTYV&llsBvSSdY^G`iP=T|KOTAw|Laj6TGnb1m#Hf!jwuWO1jN3Xmlw?#kZn)bvuZ! zb8?tpXAz4v(9v%Nc>at8=^Jm_Ph>Og7e;_al>f{eT!S$|QP`Ymh{=BDIO|6wEZHCjz1<6$_k*|K z>fVeqK6b*u9hqpkHJZD5wJ^njWsqS?jG7H{CN;KW>Gt&~R^$;U!;tA6ddY;B6(VV? zsbYk`3hYm~qEY`l5FFBgbzaGk`soxZ6W54Dy;p+Exe!on8jqb*-sAi+#M06W#W7F9 zF!aA~P;Y`L9<~@d@5f?#PBYf%Gn5A;DKg8OSjudF@aQk6-pqCks;g!rt`$MQab=M3 z*o5Ato>()YigK>Az&B&~ zCA{Ijd(VN+AC|=C&t!V zeIOsE&%Z5C$CPK(rT3&+h0|l6lJz(BB3-dL#~zi-DG&2!UzTFjA04kA;hI|w7-m=x zDK`1|;QTG{?{Nflmc((t`G;A_vLe(PbqQ*t&Ou?49s1pS$u`bC4)(&4DE?@`^sXz& zbs50SBICF?hIo;$X^*mjavV!T(RZpJd9O-A7eYYZK9G0U=Yed4A0*^G0P~JKY#H?$ z1uc_ke&z#~kz$PBRtw&MYgJV1NJW9&`-N&{26(;m8j**BbFr}jhW`XKZ)83o0a|df-c28fs3Y&jG=eJhAXILRP zOzUBHX5>X!n}oEZ8hLX=lNo4hUV+nUJ-eercEH=ngZaNV-o`iUcEi7;R^XwBMew+- z6s@+oV}HlK5b}JgU4z>zcu+i8v8-&BUF@+7EdQWCe{Z%Bcg|^q$&(hNj?b!_CIe2w zz^Xdn2fgePL&IQY?r2=}V=Z`<>e=PbJ_&<{ros6%=bJ~VGv{6~%&tMl%&z;Q00&6M z;@^R@Z!Srv+1vO`IFObCC7w}u&c_4hPCms7W*XS-->QVYUTbfLE}v_6*fKP`HtFW;wF(Fx`W!Z`7-v`e-&~ly-vI~HPJ8aQJH!P)4Wv7PLfK|~ z)AG8hKj`I#JgARezf(>#K6xwdKKD#xKN0OIJ}=P)zAitOKBOk3KB>%-KHD;@ oJ|yfqzhz8{KCCbVKMn{VKg_39K6b$jzJoBBzM@@(K973nJ_#OkQUCw| literal 0 HcmV?d00001 diff --git a/tests/test_convert.py b/tests/test_convert.py index 816b163a..ff51abcb 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -3,7 +3,14 @@ import torch import torch.nn.functional as F -from chatglm_cpp.convert import quantize_q4_0, quantize_q4_1, quantize_q5_0, quantize_q5_1, quantize_q8_0 +from chatglm_cpp.convert import ( + get_prefix_cache, + quantize_q4_0, + quantize_q4_1, + quantize_q5_0, + quantize_q5_1, + quantize_q8_0, +) HERE = Path(__file__).resolve().parent @@ -168,10 +175,6 @@ def test_quantize_q5_1(): assert (q_tensor == ggml_q_tensor).all() -CHATGLM_MODEL_PATH = Path( - "~/.cache/huggingface/hub/models--THUDM--chatglm-6b/snapshots/619e736c6d4cd139840579c5482063b75bed5666" -).expanduser() - CHATGLM2_MODEL_PATH = Path( "~/.cache/huggingface/hub/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f" ).expanduser() @@ -242,6 +245,44 @@ def make_data_rms_norm(): def make_data_glm_model(): + def _forward_steps(model, seq_len): + # self attention + x1 = torch.arange(seq_len, dtype=torch.int64)[None, :] + position_ids = torch.tensor([[[0, 1, 1], [0, 0, 1]]]) + attn_mask = torch.tensor([[0, 0, 1], [0, 0, 1], [0, 0, 0]], dtype=torch.bool)[None, None, :] + with torch.no_grad(): + out = model(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True) + y1 = out.last_hidden_state + kv_cache = out.past_key_values + + # cross attention + x2 = torch.tensor([[seq_len]], dtype=torch.int64) + position_ids = torch.tensor([[[1], [2]]]) + attn_mask = None + with torch.no_grad(): + out = model( + x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True + ) + y2 = out.last_hidden_state + kv_cache = out.past_key_values + + # cross attention + x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64) + position_ids = torch.tensor([[[1], [3]]]) + attn_mask = None + with torch.no_grad(): + out = model( + x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True + ) + y3 = out.last_hidden_state + kv_cache = out.past_key_values + + return x1, y1, x2, y2, x3, y3 + + CHATGLM_MODEL_PATH = Path( + "~/.cache/huggingface/hub/models--THUDM--chatglm-6b/snapshots/8b7d33596d18c5e83e2da052d05ca4db02e60620" + ).expanduser() + sys.path.append(str(CHATGLM_MODEL_PATH)) from modeling_chatglm import ChatGLMModel from transformers import AutoConfig @@ -260,36 +301,54 @@ def make_data_glm_model(): seq_len = 3 - # self attention - x1 = torch.arange(seq_len, dtype=torch.int64)[None, :] - position_ids = torch.tensor([[[0, 1, 1], [0, 0, 1]]]) - attn_mask = torch.tensor([[0, 0, 1], [0, 0, 1], [0, 0, 0]], dtype=torch.bool)[None, None, :] - with torch.no_grad(): - out = m(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True) - y1 = out.last_hidden_state - kv_cache = out.past_key_values + x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len) - # cross attention - x2 = torch.tensor([[seq_len]], dtype=torch.int64) - position_ids = torch.tensor([[[1], [2]]]) - attn_mask = None - with torch.no_grad(): - out = m(x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True) - y2 = out.last_hidden_state - kv_cache = out.past_key_values + print(m) - # cross attention - x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64) - position_ids = torch.tensor([[[1], [3]]]) - attn_mask = None - with torch.no_grad(): - out = m(x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True) - y3 = out.last_hidden_state - kv_cache = out.past_key_values + with open(HERE / "data/glm_model.data", "wb") as f: + m.word_embeddings.weight.data.numpy().tofile(f) + m.layers[0].input_layernorm.weight.data.numpy().tofile(f) + m.layers[0].input_layernorm.bias.data.numpy().tofile(f) + m.layers[0].attention.query_key_value.weight.data.numpy().tofile(f) + m.layers[0].attention.query_key_value.bias.data.numpy().tofile(f) + m.layers[0].attention.dense.weight.data.numpy().tofile(f) + m.layers[0].attention.dense.bias.data.numpy().tofile(f) + m.layers[0].post_attention_layernorm.weight.data.numpy().tofile(f) + m.layers[0].post_attention_layernorm.bias.data.numpy().tofile(f) + m.layers[0].mlp.dense_h_to_4h.weight.data.numpy().tofile(f) + m.layers[0].mlp.dense_h_to_4h.bias.data.numpy().tofile(f) + m.layers[0].mlp.dense_4h_to_h.weight.data.numpy().tofile(f) + m.layers[0].mlp.dense_4h_to_h.bias.data.numpy().tofile(f) + m.final_layernorm.weight.data.numpy().tofile(f) + m.final_layernorm.bias.data.numpy().tofile(f) + + x1.int().numpy().tofile(f) + y1.data.numpy().tofile(f) + x2.int().numpy().tofile(f) + y2.data.numpy().tofile(f) + x3.int().numpy().tofile(f) + y3.data.numpy().tofile(f) + + # p-tuning v2 + config.pre_seq_len = 5 + m = ChatGLMModel(config).float().eval() + for param in m.parameters(): + param.data.uniform_(-0.5, 0.5) + + x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len) print(m) - with open(HERE / "data/glm_model.data", "wb") as f: + past_key_values = get_prefix_cache( + m.prefix_encoder, + config.pre_seq_len, + config.num_layers, + config.num_attention_heads, + config.hidden_size // config.num_attention_heads, + ) + + with open(HERE / "data/glm_ptuning_v2_model.data", "wb") as f: + past_key_values.data.numpy().tofile(f) m.word_embeddings.weight.data.numpy().tofile(f) m.layers[0].input_layernorm.weight.data.numpy().tofile(f) m.layers[0].input_layernorm.bias.data.numpy().tofile(f) @@ -385,7 +444,44 @@ def make_data_glm2_model(): def make_data_glm3_model(): - CHATGLM3_MODEL_PATH = Path("./chatglm3-6b").expanduser() + + def _forward_steps(model, seq_len): + # self attention + x1 = torch.arange(seq_len, dtype=torch.int64)[None, :] + position_ids = torch.arange(seq_len, dtype=torch.int64)[None, :] + attn_mask = torch.ones(1, seq_len, dtype=torch.int64) + with torch.no_grad(): + out = model(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True) + y1 = out.last_hidden_state + kv_cache = out.past_key_values + + # cross attention + x2 = torch.tensor([[seq_len]], dtype=torch.int64) + position_ids = torch.tensor([[seq_len]], dtype=torch.int64) + attn_mask = torch.ones(1, seq_len + 1, dtype=torch.int64) + with torch.no_grad(): + out = model( + x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True + ) + y2 = out.last_hidden_state + kv_cache = out.past_key_values + + # cross attention + x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64) + position_ids = torch.tensor([[seq_len + 1]], dtype=torch.int64) + attn_mask = torch.ones(1, seq_len + 2, dtype=torch.int64) + with torch.no_grad(): + out = model( + x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True + ) + y3 = out.last_hidden_state + kv_cache = out.past_key_values + + return x1, y1, x2, y2, x3, y3 + + CHATGLM3_MODEL_PATH = Path( + "~/.cache/huggingface/hub/models--THUDM--chatglm3-6b/snapshots/a5ba5501eb873d40d48bd0983bd2a8dd006bb838" + ).expanduser() sys.path.append(str(CHATGLM3_MODEL_PATH)) from modeling_chatglm import ChatGLMModel @@ -407,36 +503,44 @@ def make_data_glm3_model(): seq_len = 3 - # self attention - x1 = torch.arange(seq_len, dtype=torch.int64)[None, :] - position_ids = torch.arange(seq_len, dtype=torch.int64)[None, :] - attn_mask = torch.ones(1, seq_len, dtype=torch.int64) - with torch.no_grad(): - out = m(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True) - y1 = out.last_hidden_state - kv_cache = out.past_key_values + x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len) - # cross attention - x2 = torch.tensor([[seq_len]], dtype=torch.int64) - position_ids = torch.tensor([[seq_len]], dtype=torch.int64) - attn_mask = torch.ones(1, seq_len + 1, dtype=torch.int64) - with torch.no_grad(): - out = m(x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True) - y2 = out.last_hidden_state - kv_cache = out.past_key_values + print(m) - # cross attention - x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64) - position_ids = torch.tensor([[seq_len + 1]], dtype=torch.int64) - attn_mask = torch.ones(1, seq_len + 2, dtype=torch.int64) - with torch.no_grad(): - out = m(x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True) - y3 = out.last_hidden_state - kv_cache = out.past_key_values + with open(HERE / "data/glm3_model.data", "wb") as f: + m.embedding.word_embeddings.weight.data.numpy().tofile(f) + m.encoder.layers[0].input_layernorm.weight.data.numpy().tofile(f) + m.encoder.layers[0].self_attention.query_key_value.weight.data.numpy().tofile(f) + m.encoder.layers[0].self_attention.query_key_value.bias.data.numpy().tofile(f) + m.encoder.layers[0].self_attention.dense.weight.data.numpy().tofile(f) + m.encoder.layers[0].post_attention_layernorm.weight.data.numpy().tofile(f) + m.encoder.layers[0].mlp.dense_h_to_4h.weight.data.numpy().tofile(f) + m.encoder.layers[0].mlp.dense_4h_to_h.weight.data.numpy().tofile(f) + m.encoder.final_layernorm.weight.data.numpy().tofile(f) + + x1.int().numpy().tofile(f) + y1.numpy().tofile(f) + x2.int().numpy().tofile(f) + y2.numpy().tofile(f) + x3.int().numpy().tofile(f) + y3.numpy().tofile(f) + + # p-tuning v2 + config.pre_seq_len = 5 + m = ChatGLMModel(config).float().eval() + for param in m.parameters(): + param.data.uniform_(-0.5, 0.5) + + x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len) print(m) - with open(HERE / "data/glm3_model.data", "wb") as f: + past_key_values = get_prefix_cache( + m.prefix_encoder, config.pre_seq_len, config.num_layers, config.multi_query_group_num, config.kv_channels + ) + + with open(HERE / "data/glm3_ptuning_v2_model.data", "wb") as f: + past_key_values.data.numpy().tofile(f) m.embedding.word_embeddings.weight.data.numpy().tofile(f) m.encoder.layers[0].input_layernorm.weight.data.numpy().tofile(f) m.encoder.layers[0].self_attention.query_key_value.weight.data.numpy().tofile(f)