Last active
June 24, 2024 16:02
-
-
Save etcetra7n/702bb400b9b35f00f36a96354384bb62 to your computer and use it in GitHub Desktop.
Implementation of the original transformer model described by Vaswani et al for English to German translation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"id": "934121cc-1ad8-41da-8653-997bafa88d8b", | |
"metadata": {}, | |
"source": [ | |
"# Implementation of original transformer model as described by Vaswani et al for English to German translation\n", | |
"\n", | |
"This is the implementation of the original transformer model described by Vaswani et al in the paper \"Attention is all you need\", trained for translation of English sentences to German, implemented using tensorflow and keras. It is trained on a small dataset consisting of about 150000 English to German sentence pairs. It features all the elements described in the paper including Mulltihead Attention mechanism, Positional Encoding and a learning rate scheduler. However due to limitations of computation resources and small size of dataset, the model currently does not provide accurate translation. But the reader may feel free to play with the model, suggest any improvements or train the model on a better training dataset" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"id": "137b57fa-4bdb-4798-b57e-209f3f82a02e", | |
"metadata": { | |
"colab": { | |
"base_uri": "https://localhost:8080/" | |
}, | |
"collapsed": true, | |
"executionInfo": { | |
"elapsed": 10076, | |
"status": "ok", | |
"timestamp": 1719240582183, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "137b57fa-4bdb-4798-b57e-209f3f82a02e", | |
"jupyter": { | |
"outputs_hidden": true, | |
"source_hidden": true | |
}, | |
"outputId": "9dbc5e1b-5a8e-4cab-fd84-c499fb56e6aa", | |
"scrolled": true, | |
"tags": [] | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Saved: data/english-german.pkl\n" | |
] | |
} | |
], | |
"source": [ | |
"# Dataset Cleaning\n", | |
"import re\n", | |
"import string\n", | |
"from unicodedata import normalize\n", | |
"from numpy import array\n", | |
"from pickle import dump\n", | |
"\n", | |
"def load_doc(filename):\n", | |
" file = open(filename, mode='rt', encoding='utf-8')\n", | |
" text = file.read()\n", | |
" file.close()\n", | |
" return text\n", | |
"def to_pairs(doc):\n", | |
" lines = doc.strip().split('\\n')\n", | |
" pairs = [line.split('\\t') for line in lines]\n", | |
" return pairs\n", | |
"def clean_pairs(lines):\n", | |
" cleaned = list()\n", | |
" re_print = re.compile('[^%s]' % re.escape(string.printable))\n", | |
" table = str.maketrans('', '', string.punctuation)\n", | |
" for pair in lines:\n", | |
" clean_pair = list()\n", | |
" for line in pair:\n", | |
" line = normalize('NFD', line).encode('ascii', 'ignore')\n", | |
" line = line.decode('UTF-8')\n", | |
" line = line.split()\n", | |
" line = [word.lower() for word in line]\n", | |
" line = [word.translate(table) for word in line] # remove punctuations\n", | |
" line = [re_print.sub('', w) for w in line] # remove non printable characters\n", | |
" line = [word for word in line if word.isalpha()] # remove numbers\n", | |
" clean_pair.append(' '.join(line))\n", | |
" cleaned.append(clean_pair)\n", | |
" return array(cleaned)\n", | |
"def save_clean_data(sentences, filename):\n", | |
" dump(sentences, open(filename, 'wb'))\n", | |
" print('Saved: %s' % filename)\n", | |
"filename = 'data/deu.txt'\n", | |
"doc = load_doc(filename)\n", | |
"pairs = to_pairs(doc)\n", | |
"clean_pairs = clean_pairs(pairs)\n", | |
"save_clean_data(clean_pairs, 'data/english-german.pkl')" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"id": "056422de-17ab-4cd6-a843-98e0e3eab263", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 3623, | |
"status": "ok", | |
"timestamp": 1719241658965, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "056422de-17ab-4cd6-a843-98e0e3eab263", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Prepare dataset\n", | |
"from pickle import load, dump, HIGHEST_PROTOCOL\n", | |
"from numpy import savetxt\n", | |
"from pickle import load\n", | |
"from numpy.random import shuffle\n", | |
"from keras.preprocessing.text import Tokenizer\n", | |
"from keras.preprocessing.sequence import pad_sequences\n", | |
"from tensorflow import convert_to_tensor, int64\n", | |
"\n", | |
"class PrepareDataset:\n", | |
" def __init__(self, **kwargs):\n", | |
" super(PrepareDataset, self).__init__(**kwargs)\n", | |
" self.n_sentences = 12000\n", | |
" self.train_split = 0.8\n", | |
" self.val_split = 0.2\n", | |
"\n", | |
" def create_tokenizer(self, dataset):\n", | |
" tokenizer = Tokenizer()\n", | |
" tokenizer.fit_on_texts(dataset)\n", | |
" return tokenizer\n", | |
"\n", | |
" def find_seq_length(self, dataset):\n", | |
" return max(len(seq.split()) for seq in dataset)\n", | |
"\n", | |
" def find_vocab_size(self, tokenizer, dataset):\n", | |
" tokenizer.fit_on_texts(dataset)\n", | |
" return len(tokenizer.word_index) + 1\n", | |
"\n", | |
" def save_tokenizer(self, tokenizer_tuple, name):\n", | |
" with open(f\"tokenizer/{name}_tokenizer.pkl\", 'wb') as handle:\n", | |
" dump(tokenizer_tuple, handle, protocol=HIGHEST_PROTOCOL)\n", | |
"\n", | |
" def __call__(self, filename, **kwargs):\n", | |
" clean_dataset = load(open(filename, 'rb'))\n", | |
" dataset = clean_dataset[:self.n_sentences, :]\n", | |
" for i in range(dataset[:, 0].size):\n", | |
" dataset[i, 0] = \"<START> \" + dataset[i, 0] + \" <EOS>\"\n", | |
" dataset[i, 1] = \"<START> \" + dataset[i, 1] + \" <EOS>\"\n", | |
" shuffle(dataset)\n", | |
" train = dataset[:int(self.n_sentences * self.train_split)]\n", | |
" val = dataset[int(self.n_sentences * self.train_split):int(self.n_sentences * self.train_split)+int(self.n_sentences * (self.val_split))]\n", | |
"\n", | |
" # Prepare tokenizer for the encoder input\n", | |
" enc_tokenizer = self.create_tokenizer(train[:, 0])\n", | |
" enc_seq_length = self.find_seq_length(train[:, 0])\n", | |
" enc_vocab_size = self.find_vocab_size(enc_tokenizer, train[:, 0])\n", | |
"\n", | |
" # Encode and pad the input sequences\n", | |
" trainX = enc_tokenizer.texts_to_sequences(train[:, 0])\n", | |
" trainX = pad_sequences(trainX, maxlen=enc_seq_length, padding='post')\n", | |
" trainX = convert_to_tensor(trainX, dtype=int64)\n", | |
"\n", | |
" # Prepare tokenizer for the decoder input\n", | |
" dec_tokenizer = self.create_tokenizer(train[:, 1])\n", | |
" dec_seq_length = self.find_seq_length(train[:, 1])\n", | |
" dec_vocab_size = self.find_vocab_size(dec_tokenizer, train[:, 1])\n", | |
"\n", | |
" # Encode and pad the input sequences\n", | |
" trainY = dec_tokenizer.texts_to_sequences(train[:, 1])\n", | |
" trainY = pad_sequences(trainY, maxlen=dec_seq_length, padding='post')\n", | |
" trainY = convert_to_tensor(trainY, dtype=int64)\n", | |
"\n", | |
" # Validation dataset\n", | |
" valX = enc_tokenizer.texts_to_sequences(val[:, 0])\n", | |
" valX = pad_sequences(valX, maxlen=enc_seq_length, padding='post')\n", | |
" valX = convert_to_tensor(valX, dtype=int64)\n", | |
"\n", | |
" valY = dec_tokenizer.texts_to_sequences(val[:, 1])\n", | |
" valY = pad_sequences(valY, maxlen=dec_seq_length, padding='post')\n", | |
" valY = convert_to_tensor(valY, dtype=int64)\n", | |
"\n", | |
" # Save the encoder tokenizer\n", | |
" self.save_tokenizer((enc_tokenizer, enc_seq_length, enc_vocab_size), 'enc')\n", | |
"\n", | |
" # Save the decoder tokenizer\n", | |
" self.save_tokenizer((dec_tokenizer, dec_seq_length, dec_vocab_size), 'dec')\n", | |
"\n", | |
" return trainX, trainY, valX, valY, train, enc_seq_length, dec_seq_length, enc_vocab_size, dec_vocab_size" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"id": "ca525f41-074a-4295-9d00-bf8ebd424942", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 633, | |
"status": "ok", | |
"timestamp": 1719241668164, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "ca525f41-074a-4295-9d00-bf8ebd424942", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Multi Head Attention\n", | |
"\n", | |
"import tensorflow as tf\n", | |
"from tensorflow import math, matmul, reshape, shape, transpose, cast, float32\n", | |
"from tensorflow.keras.layers import Dense, Layer\n", | |
"from tensorflow.keras.backend import softmax\n", | |
"\n", | |
"# Implementing the Scaled-Dot Product Attention\n", | |
"class DotProductAttention(Layer):\n", | |
" def __init__(self, **kwargs):\n", | |
" super(DotProductAttention, self).__init__(**kwargs)\n", | |
"\n", | |
" def call(self, queries, keys, values, d_k, mask=None):\n", | |
" # Scoring the queries against the keys after transposing the latter, and scaling\n", | |
" scores = matmul(queries, keys, transpose_b=True) / math.sqrt(cast(d_k, float32))\n", | |
"\n", | |
" # Apply mask to the attention scores\n", | |
" if mask is not None:\n", | |
" scores += -1e9 * mask\n", | |
"\n", | |
" # Computing the weights by a softmax operation\n", | |
" weights = softmax(scores)\n", | |
"\n", | |
" # Computing the attention by a weighted sum of the value vectors\n", | |
" return matmul(weights, values)\n", | |
"\n", | |
"class MultiHeadAttention(Layer):\n", | |
" def __init__(self, h, d_k, d_v, d_model, **kwargs):\n", | |
" super(MultiHeadAttention, self).__init__(**kwargs)\n", | |
" self.attention = DotProductAttention() # Scaled dot product attention\n", | |
" self.heads = h # Number of attention heads to use\n", | |
" self.d_k = d_k # Dimensionality of the linearly projected queries and keys\n", | |
" self.d_v = d_v # Dimensionality of the linearly projected values\n", | |
" self.d_model = d_model # Dimensionality of the model\n", | |
" self.W_q = Dense(d_k) # Learned projection matrix for the queries\n", | |
" self.W_k = Dense(d_k) # Learned projection matrix for the keys\n", | |
" self.W_v = Dense(d_v) # Learned projection matrix for the values\n", | |
" self.W_o = Dense(d_model) # Learned projection matrix for the multi-head output\n", | |
"\n", | |
" def reshape_tensor(self, x, heads, flag):\n", | |
" if flag:\n", | |
" # Tensor shape after reshaping and transposing: (batch_size, heads, seq_length, -1)\n", | |
" x = reshape(x, shape=(shape(x)[0], shape(x)[1], heads, -1))\n", | |
" x = transpose(x, perm=(0, 2, 1, 3))\n", | |
" else:\n", | |
" # Reverting the reshaping and transposing operations: (batch_size, seq_length, d_k)\n", | |
" x = transpose(x, perm=(0, 2, 1, 3))\n", | |
" x = reshape(x, shape=(shape(x)[0], shape(x)[1], self.d_k))\n", | |
" return x\n", | |
"\n", | |
" def call(self, queries, keys, values, mask=None):\n", | |
" # Rearrange the queries to be able to compute all heads in parallel\n", | |
" q_reshaped = self.reshape_tensor(self.W_q(queries), self.heads, True)\n", | |
" # Resulting tensor shape: (batch_size, heads, input_seq_length, -1)\n", | |
"\n", | |
" # Rearrange the keys to be able to compute all heads in parallel\n", | |
" k_reshaped = self.reshape_tensor(self.W_k(keys), self.heads, True)\n", | |
" # Resulting tensor shape: (batch_size, heads, input_seq_length, -1)\n", | |
"\n", | |
" # Rearrange the values to be able to compute all heads in parallel\n", | |
" v_reshaped = self.reshape_tensor(self.W_v(values), self.heads, True)\n", | |
" # Resulting tensor shape: (batch_size, heads, input_seq_length, -1)\n", | |
"\n", | |
" # Compute the multi-head attention output using the reshaped queries, keys and values\n", | |
" #print(f\"{type(self.d_k)} {self.d_k}\")\n", | |
" #print(f\"{type(mask)} {mask}\")\n", | |
" o_reshaped = self.attention(q_reshaped, k_reshaped, v_reshaped, mask=mask, d_k=self.d_k) #\n", | |
" # Resulting tensor shape: (batch_size, heads, input_seq_length, -1)\n", | |
"\n", | |
" # Rearrange back the output into concatenated form\n", | |
" output = self.reshape_tensor(o_reshaped, self.heads, False)\n", | |
" # Resulting tensor shape: (batch_size, input_seq_length, d_v)\n", | |
"\n", | |
" # Apply one final linear projection to the output to generate the multi-head attention\n", | |
" # Resulting tensor shape: (batch_size, input_seq_length, d_model)\n", | |
" return self.W_o(output)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"id": "35f497a0-4f9e-4faa-bbdb-0b2c40c95fd6", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 525, | |
"status": "ok", | |
"timestamp": 1719241673326, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "35f497a0-4f9e-4faa-bbdb-0b2c40c95fd6", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Position Embedding\n", | |
"\n", | |
"import tensorflow as tf\n", | |
"from tensorflow.keras.layers import Embedding, Layer\n", | |
"import numpy as np\n", | |
"'''\n", | |
"class PositionEmbeddingLayer(Layer):\n", | |
" def __init__(self, sequence_length, vocab_size, output_dim, **kwargs):\n", | |
" super(PositionEmbeddingLayer, self).__init__(**kwargs)\n", | |
" self.word_embedding_layer = Embedding(\n", | |
" input_dim=vocab_size, output_dim=output_dim\n", | |
" )\n", | |
" self.position_embedding_layer = Embedding(\n", | |
" input_dim=sequence_length, output_dim=output_dim\n", | |
" )\n", | |
"\n", | |
" def call(self, inputs):\n", | |
" position_indices = tf.range(tf.shape(inputs)[-1])\n", | |
" embedded_words = self.word_embedding_layer(inputs)\n", | |
" embedded_indices = self.position_embedding_layer(position_indices)\n", | |
" return embedded_words + embedded_indices\n", | |
"'''\n", | |
"class PositionEmbeddingFixedWeights(Layer):\n", | |
" def __init__(self, sequence_length, vocab_size, output_dim, **kwargs):\n", | |
" super(PositionEmbeddingFixedWeights, self).__init__(**kwargs)\n", | |
" word_embedding_matrix = self.get_position_encoding(vocab_size, output_dim)\n", | |
" position_embedding_matrix = self.get_position_encoding(sequence_length, output_dim)\n", | |
" self.word_embedding_layer = Embedding(\n", | |
" input_dim=vocab_size, output_dim=output_dim,\n", | |
" weights=[word_embedding_matrix],\n", | |
" trainable=False\n", | |
" )\n", | |
" self.position_embedding_layer = Embedding(\n", | |
" input_dim=sequence_length, output_dim=output_dim,\n", | |
" weights=[position_embedding_matrix],\n", | |
" trainable=False\n", | |
" )\n", | |
" def get_position_encoding(self, seq_len, d, n=10000):\n", | |
" P = np.zeros((seq_len, d))\n", | |
" for k in range(seq_len):\n", | |
" for i in np.arange(int(d/2)):\n", | |
" denominator = np.power(n, 2*i/d)\n", | |
" P[k, 2*i] = np.sin(k/denominator)\n", | |
" P[k, 2*i+1] = np.cos(k/denominator)\n", | |
" return P\n", | |
"\n", | |
" def call(self, inputs):\n", | |
" position_indices = tf.range(tf.shape(inputs)[-1])\n", | |
" embedded_words = self.word_embedding_layer(inputs)\n", | |
" embedded_indices = self.position_embedding_layer(position_indices)\n", | |
" return embedded_words + embedded_indices" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"id": "4f2341a4-4930-4f40-847e-830041c15e9f", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 510, | |
"status": "ok", | |
"timestamp": 1719241677692, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "4f2341a4-4930-4f40-847e-830041c15e9f", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"scrolled": true, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Encoder\n", | |
"\n", | |
"from tensorflow.keras.layers import LayerNormalization, Layer, Dense, ReLU, Dropout\n", | |
"import keras\n", | |
"\n", | |
"# Implementing the Add & Norm Layer\n", | |
"class AddNormalization(Layer):\n", | |
" def __init__(self, **kwargs):\n", | |
" super(AddNormalization, self).__init__(**kwargs)\n", | |
" self.layer_norm = LayerNormalization() # Layer normalization layer\n", | |
"\n", | |
" def call(self, x, sublayer_x):\n", | |
" add = x + sublayer_x\n", | |
" return self.layer_norm(add)\n", | |
"\n", | |
"# Implementing the Feed-Forward Layer\n", | |
"class FeedForward(Layer):\n", | |
" def __init__(self, d_ff, d_model, **kwargs):\n", | |
" super(FeedForward, self).__init__(**kwargs)\n", | |
" self.fully_connected1 = Dense(d_ff) # First fully connected layer\n", | |
" self.fully_connected2 = Dense(d_model) # Second fully connected layer\n", | |
" self.activation = ReLU() # ReLU activation layer\n", | |
"\n", | |
" def call(self, x):\n", | |
" # The input is passed into the two fully-connected layers, with a ReLU in between\n", | |
" x_fc1 = self.fully_connected1(x)\n", | |
" return self.fully_connected2(self.activation(x_fc1))\n", | |
"\n", | |
"# Implementing the Encoder Layer\n", | |
"@keras.saving.register_keras_serializable()\n", | |
"class EncoderLayer(Layer):\n", | |
" def __init__(self, h, d_k, d_v, d_model, d_ff, rate, **kwargs):\n", | |
" super(EncoderLayer, self).__init__(**kwargs)\n", | |
" self.multihead_attention = MultiHeadAttention(h, d_k, d_v, d_model)\n", | |
" self.dropout1 = Dropout(rate)\n", | |
" self.add_norm1 = AddNormalization()\n", | |
" self.feed_forward = FeedForward(d_ff, d_model)\n", | |
" self.dropout2 = Dropout(rate)\n", | |
" self.add_norm2 = AddNormalization()\n", | |
"\n", | |
" def call(self, x, padding_mask, training):\n", | |
" # Multi-head attention layer\n", | |
" multihead_output = self.multihead_attention(x, x, x, padding_mask)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Add in a dropout layer\n", | |
" multihead_output = self.dropout1(multihead_output, training=training)\n", | |
"\n", | |
" # Followed by an Add & Norm layer\n", | |
" addnorm_output = self.add_norm1(x, multihead_output)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Followed by a fully connected layer\n", | |
" feedforward_output = self.feed_forward(addnorm_output)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Add in another dropout layer\n", | |
" feedforward_output = self.dropout2(feedforward_output, training=training)\n", | |
"\n", | |
" # Followed by another Add & Norm layer\n", | |
" return self.add_norm2(addnorm_output, feedforward_output)\n", | |
"\n", | |
"class Encoder(Layer):\n", | |
" def __init__(self, vocab_size, sequence_length, h, d_k, d_v, d_model, d_ff, n, rate, **kwargs):\n", | |
" super(Encoder, self).__init__(**kwargs)\n", | |
" self.pos_encoding = PositionEmbeddingFixedWeights(sequence_length, vocab_size, d_model)\n", | |
" self.dropout = Dropout(rate)\n", | |
" self.encoder_layer = [EncoderLayer(h, d_k, d_v, d_model, d_ff, rate) for _ in range(n)]\n", | |
"\n", | |
" def call(self, input_sentence, padding_mask, training):\n", | |
" # Generate the positional encoding\n", | |
" pos_encoding_output = self.pos_encoding(input_sentence)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Add in a dropout layer\n", | |
" x = self.dropout(pos_encoding_output, training=training)\n", | |
"\n", | |
" # Pass on the positional encoded values to each encoder layer\n", | |
" for i, layer in enumerate(self.encoder_layer):\n", | |
" x = layer(x, padding_mask, training=training)\n", | |
"\n", | |
" return x" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"id": "fab81cec-009b-405f-99d8-cc25680ea4d2", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 6, | |
"status": "ok", | |
"timestamp": 1719241679740, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "fab81cec-009b-405f-99d8-cc25680ea4d2", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Decoder\n", | |
"\n", | |
"from tensorflow.keras.layers import Layer, Dropout\n", | |
"import tensorflow as tf\n", | |
"\n", | |
"# Implementing the Decoder Layer\n", | |
"@tf.keras.utils.register_keras_serializable()\n", | |
"class DecoderLayer(Layer):\n", | |
" def __init__(self, h, d_k, d_v, d_model, d_ff, rate, **kwargs):\n", | |
" super(DecoderLayer, self).__init__(**kwargs)\n", | |
" self.h = h\n", | |
" self.d_k = d_k\n", | |
" self.d_v = d_v\n", | |
" self.d_model = d_model\n", | |
" self.d_ff = d_ff\n", | |
" self.rate = rate\n", | |
"\n", | |
" self.multihead_attention1 = MultiHeadAttention(h, d_k, d_v, d_model)\n", | |
" self.dropout1 = Dropout(rate)\n", | |
" self.add_norm1 = AddNormalization()\n", | |
" self.multihead_attention2 = MultiHeadAttention(h, d_k, d_v, d_model)\n", | |
" self.dropout2 = Dropout(rate)\n", | |
" self.add_norm2 = AddNormalization()\n", | |
" self.feed_forward = FeedForward(d_ff, d_model)\n", | |
" self.dropout3 = Dropout(rate)\n", | |
" self.add_norm3 = AddNormalization()\n", | |
"\n", | |
" def build(self, input_shape):\n", | |
" self.multihead_attention1 = MultiHeadAttention(self.h, self.d_k, self.d_v, self.d_model)\n", | |
" self.dropout1 = Dropout(self.rate)\n", | |
" self.add_norm1 = AddNormalization()\n", | |
" self.multihead_attention2 = MultiHeadAttention(self.h, self.d_k, self.d_v, self.d_model)\n", | |
" self.dropout2 = Dropout(self.rate)\n", | |
" self.add_norm2 = AddNormalization()\n", | |
" self.feed_forward = FeedForward(self.d_ff, self.d_model)\n", | |
" self.dropout3 = Dropout(self.rate)\n", | |
" self.add_norm3 = AddNormalization()\n", | |
"\n", | |
" def call(self, x, encoder_output, lookahead_mask, padding_mask, training):\n", | |
" # Multi-head attention layer\n", | |
" multihead_output1 = self.multihead_attention1(x, x, x, lookahead_mask)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Add in a dropout layer\n", | |
" multihead_output1 = self.dropout1(multihead_output1, training=training)\n", | |
"\n", | |
" # Followed by an Add & Norm layer\n", | |
" addnorm_output1 = self.add_norm1(x, multihead_output1)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Followed by another multi-head attention layer\n", | |
" multihead_output2 = self.multihead_attention2(addnorm_output1, encoder_output, encoder_output, padding_mask)\n", | |
"\n", | |
" # Add in another dropout layer\n", | |
" multihead_output2 = self.dropout2(multihead_output2, training=training)\n", | |
"\n", | |
" # Followed by another Add & Norm layer\n", | |
" addnorm_output2 = self.add_norm1(addnorm_output1, multihead_output2)\n", | |
"\n", | |
" # Followed by a fully connected layer\n", | |
" feedforward_output = self.feed_forward(addnorm_output2)\n", | |
" # Expected output shape = (batch_size, sequence_length, d_model)\n", | |
"\n", | |
" # Add in another dropout layer\n", | |
" feedforward_output = self.dropout3(feedforward_output, training=training)\n", | |
"\n", | |
" # Followed by another Add & Norm layer\n", | |
" return self.add_norm3(addnorm_output2, feedforward_output)\n", | |
"\n", | |
"# Implementing the Decoder\n", | |
"class Decoder(Layer):\n", | |
" def __init__(self, vocab_size, sequence_length, h, d_k, d_v, d_model, d_ff, n, rate, **kwargs):\n", | |
" super(Decoder, self).__init__(**kwargs)\n", | |
" self.pos_encoding = PositionEmbeddingFixedWeights(sequence_length, vocab_size, d_model)\n", | |
" self.dropout = Dropout(rate)\n", | |
" self.decoder_layer = [DecoderLayer(h, d_k, d_v, d_model, d_ff, rate) for _ in range(n)]\n", | |
"\n", | |
" def call(self, output_target, encoder_output, lookahead_mask, padding_mask, training):\n", | |
" # Generate the positional encoding\n", | |
" pos_encoding_output = self.pos_encoding(output_target)\n", | |
" # Expected output shape = (number of sentences, sequence_length, d_model)\n", | |
"\n", | |
" # Add in a dropout layer\n", | |
" x = self.dropout(pos_encoding_output, training=training)\n", | |
"\n", | |
" # Pass on the positional encoded values to each encoder layer\n", | |
" for i, layer in enumerate(self.decoder_layer):\n", | |
" x = layer(x, encoder_output, lookahead_mask, padding_mask, training=training)\n", | |
" return x" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 10, | |
"id": "4668a1f4-5d82-4ebe-ae9a-68f545053b4f", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 4, | |
"status": "ok", | |
"timestamp": 1719241683999, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "4668a1f4-5d82-4ebe-ae9a-68f545053b4f", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Transformer Model\n", | |
"\n", | |
"from tensorflow import math, cast, float32, linalg, ones, maximum, newaxis\n", | |
"from tensorflow.keras import Model\n", | |
"from tensorflow.keras.layers import Dense\n", | |
"import keras\n", | |
"\n", | |
"@keras.saving.register_keras_serializable()\n", | |
"class TransformerModel(Model):\n", | |
" def __init__(self, enc_vocab_size, dec_vocab_size, enc_seq_length, dec_seq_length, h, d_k, d_v, d_model, d_ff_inner, n, rate, **kwargs):\n", | |
" super(TransformerModel, self).__init__(**kwargs)\n", | |
" self.encoder = Encoder(enc_vocab_size, enc_seq_length, h, d_k, d_v, d_model, d_ff_inner, n, rate)\n", | |
" self.decoder = Decoder(dec_vocab_size, dec_seq_length, h, d_k, d_v, d_model, d_ff_inner, n, rate)\n", | |
" self.model_last_layer = Dense(dec_vocab_size)\n", | |
"\n", | |
" def padding_mask(self, input):\n", | |
" # Create mask which marks the zero padding values in the input by a 1.0\n", | |
" mask = math.equal(input, 0)\n", | |
" mask = cast(mask, float32)\n", | |
"\n", | |
" # The shape of the mask should be broadcastable to the shape\n", | |
" # of the attention weights that it will be masking later on\n", | |
" return mask[:, newaxis, newaxis, :]\n", | |
"\n", | |
" def lookahead_mask(self, shape):\n", | |
" # Mask out future entries by marking them with a 1.0\n", | |
" mask = 1 - linalg.band_part(ones((shape, shape)), -1, 0)\n", | |
" return mask\n", | |
"\n", | |
" def call(self, encoder_input, decoder_input, training):\n", | |
" # Create padding mask to mask the encoder inputs and the encoder outputs in the decoder\n", | |
" enc_padding_mask = self.padding_mask(encoder_input)\n", | |
"\n", | |
" # Create and combine padding and look-ahead masks to be fed into the decoder\n", | |
" dec_in_padding_mask = self.padding_mask(decoder_input)\n", | |
" dec_in_lookahead_mask = self.lookahead_mask(decoder_input.shape[1])\n", | |
" dec_in_lookahead_mask = maximum(dec_in_padding_mask, dec_in_lookahead_mask)\n", | |
"\n", | |
" encoder_output = self.encoder(encoder_input, enc_padding_mask, training=training)\n", | |
" decoder_output = self.decoder(decoder_input, encoder_output, dec_in_lookahead_mask, enc_padding_mask, training=training)\n", | |
" model_output = self.model_last_layer(decoder_output)\n", | |
"\n", | |
" return model_output" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 11, | |
"id": "3e703ba7-69f9-4b71-8298-c16e358fdf74", | |
"metadata": { | |
"colab": { | |
"base_uri": "https://localhost:8080/" | |
}, | |
"collapsed": true, | |
"editable": true, | |
"executionInfo": { | |
"elapsed": 474564, | |
"status": "ok", | |
"timestamp": 1719242201150, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "3e703ba7-69f9-4b71-8298-c16e358fdf74", | |
"jupyter": { | |
"outputs_hidden": true, | |
"source_hidden": true | |
}, | |
"outputId": "8a26d604-3f4b-4cc8-c4ad-1934cfeedf5a", | |
"tags": [] | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"\n", | |
"Start of epoch 1\n", | |
"Epoch 1 Step 0 Loss 8.4670 Accuracy 0.0000\n", | |
"Epoch 1 Step 50 Loss 7.6963 Accuracy 0.1234\n", | |
"Epoch 1 Step 100 Loss 7.0784 Accuracy 0.1695\n", | |
"Epoch 1: Training Accuracy 0.1908, Validation Accuracy 0.2638\n", | |
"Checkpoint saved at ./checkpoints/ckpt-1\n", | |
"Weights saved at model_weights/epoch1.ckpt\n", | |
"\n", | |
"Start of epoch 2\n", | |
"Epoch 2 Step 0 Loss 5.6056 Accuracy 0.2876\n", | |
"Epoch 2 Step 50 Loss 5.4511 Accuracy 0.2669\n", | |
"Epoch 2 Step 100 Loss 5.2847 Accuracy 0.2747\n", | |
"Epoch 2: Training Accuracy 0.2836, Validation Accuracy 0.3047\n", | |
"Checkpoint saved at ./checkpoints/ckpt-2\n", | |
"Weights saved at model_weights/epoch2.ckpt\n", | |
"\n", | |
"Start of epoch 3\n", | |
"Epoch 3 Step 0 Loss 4.6207 Accuracy 0.3344\n", | |
"Epoch 3 Step 50 Loss 4.6204 Accuracy 0.3290\n", | |
"Epoch 3 Step 100 Loss 4.5219 Accuracy 0.3437\n", | |
"Epoch 3: Training Accuracy 0.3553, Validation Accuracy 0.3885\n", | |
"Checkpoint saved at ./checkpoints/ckpt-3\n", | |
"Weights saved at model_weights/epoch3.ckpt\n", | |
"\n", | |
"Start of epoch 4\n", | |
"Epoch 4 Step 0 Loss 4.1215 Accuracy 0.4047\n", | |
"Epoch 4 Step 50 Loss 4.1009 Accuracy 0.3943\n", | |
"Epoch 4 Step 100 Loss 4.0305 Accuracy 0.4023\n", | |
"Epoch 4: Training Accuracy 0.4094, Validation Accuracy 0.4248\n", | |
"Checkpoint saved at ./checkpoints/ckpt-4\n", | |
"Weights saved at model_weights/epoch4.ckpt\n", | |
"\n", | |
"Start of epoch 5\n", | |
"Epoch 5 Step 0 Loss 3.7299 Accuracy 0.4448\n", | |
"Epoch 5 Step 50 Loss 3.7106 Accuracy 0.4369\n", | |
"Epoch 5 Step 100 Loss 3.6588 Accuracy 0.4416\n", | |
"Epoch 5: Training Accuracy 0.4462, Validation Accuracy 0.4566\n", | |
"Checkpoint saved at ./checkpoints/ckpt-5\n", | |
"Weights saved at model_weights/epoch5.ckpt\n", | |
"\n", | |
"Start of epoch 6\n", | |
"Epoch 6 Step 0 Loss 3.3861 Accuracy 0.4849\n", | |
"Epoch 6 Step 50 Loss 3.3973 Accuracy 0.4688\n", | |
"Epoch 6 Step 100 Loss 3.3527 Accuracy 0.4723\n", | |
"Epoch 6: Training Accuracy 0.4773, Validation Accuracy 0.4823\n", | |
"Checkpoint saved at ./checkpoints/ckpt-6\n", | |
"Weights saved at model_weights/epoch6.ckpt\n", | |
"\n", | |
"Start of epoch 7\n", | |
"Epoch 7 Step 0 Loss 3.1055 Accuracy 0.5284\n", | |
"Epoch 7 Step 50 Loss 3.1237 Accuracy 0.4947\n", | |
"Epoch 7 Step 100 Loss 3.0802 Accuracy 0.4969\n", | |
"Epoch 7: Training Accuracy 0.5014, Validation Accuracy 0.5022\n", | |
"Checkpoint saved at ./checkpoints/ckpt-7\n", | |
"Weights saved at model_weights/epoch7.ckpt\n", | |
"\n", | |
"Start of epoch 8\n", | |
"Epoch 8 Step 0 Loss 2.9322 Accuracy 0.5251\n", | |
"Epoch 8 Step 50 Loss 2.8628 Accuracy 0.5211\n", | |
"Epoch 8 Step 100 Loss 2.8263 Accuracy 0.5228\n", | |
"Epoch 8: Training Accuracy 0.5259, Validation Accuracy 0.5212\n", | |
"Checkpoint saved at ./checkpoints/ckpt-8\n", | |
"Weights saved at model_weights/epoch8.ckpt\n", | |
"\n", | |
"Start of epoch 9\n", | |
"Epoch 9 Step 0 Loss 2.5742 Accuracy 0.5619\n", | |
"Epoch 9 Step 50 Loss 2.6579 Accuracy 0.5355\n", | |
"Epoch 9 Step 100 Loss 2.6052 Accuracy 0.5409\n", | |
"Epoch 9: Training Accuracy 0.5437, Validation Accuracy 0.5297\n", | |
"Checkpoint saved at ./checkpoints/ckpt-9\n", | |
"Weights saved at model_weights/epoch9.ckpt\n", | |
"\n", | |
"Start of epoch 10\n", | |
"Epoch 10 Step 0 Loss 2.4283 Accuracy 0.5753\n", | |
"Epoch 10 Step 50 Loss 2.4183 Accuracy 0.5593\n", | |
"Epoch 10 Step 100 Loss 2.3782 Accuracy 0.5623\n", | |
"Epoch 10: Training Accuracy 0.5644, Validation Accuracy 0.5465\n", | |
"Checkpoint saved at ./checkpoints/ckpt-10\n", | |
"Weights saved at model_weights/epoch10.ckpt\n", | |
"\n", | |
"Start of epoch 11\n", | |
"Epoch 11 Step 0 Loss 2.1970 Accuracy 0.5953\n", | |
"Epoch 11 Step 50 Loss 2.2170 Accuracy 0.5796\n", | |
"Epoch 11 Step 100 Loss 2.1708 Accuracy 0.5845\n", | |
"Epoch 11: Training Accuracy 0.5876, Validation Accuracy 0.5619\n", | |
"Checkpoint saved at ./checkpoints/ckpt-11\n", | |
"Weights saved at model_weights/epoch11.ckpt\n", | |
"\n", | |
"Start of epoch 12\n", | |
"Epoch 12 Step 0 Loss 1.9493 Accuracy 0.6455\n", | |
"Epoch 12 Step 50 Loss 2.0009 Accuracy 0.6080\n", | |
"Epoch 12 Step 100 Loss 1.9701 Accuracy 0.6099\n", | |
"Epoch 12: Training Accuracy 0.6118, Validation Accuracy 0.5713\n", | |
"Checkpoint saved at ./checkpoints/ckpt-12\n", | |
"Weights saved at model_weights/epoch12.ckpt\n", | |
"Total time taken: 7.68 min\n", | |
"\n", | |
"Model: \"transformer_model\"\n", | |
"_________________________________________________________________\n", | |
" Layer (type) Output Shape Param # \n", | |
"=================================================================\n", | |
" encoder (Encoder) multiple 14710400 \n", | |
" \n", | |
" decoder (Decoder) multiple 16200960 \n", | |
" \n", | |
" dense_96 (Dense) multiple 2006856 \n", | |
" \n", | |
"=================================================================\n", | |
"Total params: 32918216 (125.57 MB)\n", | |
"Trainable params: 29599944 (112.91 MB)\n", | |
"Non-trainable params: 3318272 (12.66 MB)\n", | |
"_________________________________________________________________\n" | |
] | |
} | |
], | |
"source": [ | |
"# Training\n", | |
"\n", | |
"import tensorflow as tf\n", | |
"import keras\n", | |
"from tensorflow.keras.optimizers import Adam\n", | |
"from tensorflow.keras.optimizers.schedules import LearningRateSchedule\n", | |
"from tensorflow.keras.metrics import Mean\n", | |
"from tensorflow import data, train, math, reduce_sum, cast, equal, argmax, float32, GradientTape, TensorSpec, function, int64\n", | |
"from keras.losses import sparse_categorical_crossentropy\n", | |
"from time import time\n", | |
"from sys import stdout\n", | |
"\n", | |
"h = 8 # Number of self-attention heads\n", | |
"d_k = 64 # Dimensionality of the linearly projected queries and keys\n", | |
"d_v = 64 # Dimensionality of the linearly projected values\n", | |
"d_model = 512 # Dimensionality of model layers' outputs\n", | |
"d_ff = 2048 # Dimensionality of the inner fully connected layer\n", | |
"n = 6 # Number of layers in the encoder stack\n", | |
"\n", | |
"epochs = 12\n", | |
"batch_size = 64\n", | |
"beta_1 = 0.9\n", | |
"beta_2 = 0.98\n", | |
"epsilon = 1e-9\n", | |
"dropout_rate = 0.1\n", | |
"\n", | |
"# Implementing a learning rate scheduler\n", | |
"class LRScheduler(LearningRateSchedule):\n", | |
" def __init__(self, d_model, warmup_steps=4000, **kwargs):\n", | |
" super(LRScheduler, self).__init__(**kwargs)\n", | |
" self.d_model = cast(d_model, float32)\n", | |
" self.warmup_steps = warmup_steps\n", | |
"\n", | |
" def __call__(self, step):\n", | |
" step = tf.cast(step, dtype=tf.float32)\n", | |
" arg1 = tf.math.rsqrt(step)\n", | |
" arg2 = step * (self.warmup_steps ** -1.5)\n", | |
" return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n", | |
"\n", | |
"optimizer = Adam(LRScheduler(d_model), beta_1, beta_2, epsilon)\n", | |
"dataset = PrepareDataset()\n", | |
"trainX, trainY, valX, valY, train_orig, enc_seq_length, dec_seq_length, enc_vocab_size, dec_vocab_size = dataset('data/english-german.pkl')\n", | |
"\n", | |
"# Prepare the dataset batches\n", | |
"train_dataset = data.Dataset.from_tensor_slices((trainX, trainY))\n", | |
"train_dataset = train_dataset.batch(batch_size)\n", | |
"\n", | |
"val_dataset = data.Dataset.from_tensor_slices((valX, valY))\n", | |
"val_dataset = val_dataset.batch(batch_size)\n", | |
"\n", | |
"# Create model\n", | |
"training_model = TransformerModel(enc_vocab_size, dec_vocab_size, enc_seq_length, dec_seq_length, h, d_k, d_v, d_model, d_ff, n, dropout_rate)\n", | |
"\n", | |
"# Loss function\n", | |
"def loss_fcn(target, prediction):\n", | |
" padding_mask = math.logical_not(equal(target, 0))\n", | |
" padding_mask = cast(padding_mask, float32)\n", | |
" loss = sparse_categorical_crossentropy(target, prediction, from_logits=True) * padding_mask\n", | |
" return reduce_sum(loss) / reduce_sum(padding_mask)\n", | |
"\n", | |
"# Accuracy function\n", | |
"def accuracy_fcn(target, prediction):\n", | |
" padding_mask = math.logical_not(equal(target, 0))\n", | |
" accuracy = equal(target, argmax(prediction, axis=2))\n", | |
" accuracy = math.logical_and(padding_mask, accuracy)\n", | |
" padding_mask = cast(padding_mask, float32)\n", | |
" accuracy = cast(accuracy, float32)\n", | |
" return reduce_sum(accuracy) / reduce_sum(padding_mask)\n", | |
"\n", | |
"# Include metrics monitoring\n", | |
"train_loss = Mean(name='train_loss')\n", | |
"train_accuracy = Mean(name='train_accuracy')\n", | |
"val_loss = Mean(name='val_loss')\n", | |
"val_accuracy = Mean(name='val_accuracy')\n", | |
"\n", | |
"ckpt = train.Checkpoint(model=training_model, optimizer=optimizer)\n", | |
"ckpt_manager = train.CheckpointManager(ckpt, \"./checkpoints\", max_to_keep=3)\n", | |
"\n", | |
"# Initialise dictionaries to store the training and validation losses\n", | |
"train_loss_dict = {}\n", | |
"val_loss_dict = {}\n", | |
"train_accuracy_dict = {}\n", | |
"val_accuracy_dict = {}\n", | |
"\n", | |
"# Speeding up the training process\n", | |
"@tf.function\n", | |
"def train_step(encoder_input, decoder_input, decoder_output):\n", | |
" with GradientTape() as tape:\n", | |
" prediction = training_model(encoder_input, decoder_input, training=True)\n", | |
" loss = loss_fcn(decoder_output, prediction)\n", | |
" accuracy = accuracy_fcn(decoder_output, prediction)\n", | |
"\n", | |
" gradients = tape.gradient(loss, training_model.trainable_weights) #calculate gradients\n", | |
" optimizer.apply_gradients(zip(gradients, training_model.trainable_weights)) # Update trainable parameters\n", | |
" train_loss(loss)\n", | |
" train_accuracy(accuracy)\n", | |
"\n", | |
"start_time = time()\n", | |
"try:\n", | |
" for epoch in range(epochs):\n", | |
" train_loss.reset_state()\n", | |
" train_accuracy.reset_state()\n", | |
"\n", | |
" print(\"\\nStart of epoch %d\" % (epoch + 1))\n", | |
"\n", | |
" for step, (train_batchX, train_batchY) in enumerate(train_dataset):\n", | |
" encoder_input = train_batchX[:, 1:]\n", | |
" decoder_input = train_batchY[:, :-1]\n", | |
" decoder_output = train_batchY[:, 1:]\n", | |
" train_step(encoder_input, decoder_input, decoder_output)\n", | |
"\n", | |
" if step % 50 == 0:\n", | |
" print(f'Epoch {epoch + 1} Step {step} Loss {train_loss.result():.4f} Accuracy {train_accuracy.result():.4f}')\n", | |
" stdout.flush()\n", | |
" # print(\"Samples so far: %s\" % ((step + 1) * batch_size))\n", | |
"\n", | |
" # Run a validation step after every epoch of training\n", | |
" if (epoch+1)%1==0:\n", | |
" val_loss.reset_state()\n", | |
" val_accuracy.reset_state()\n", | |
" for val_batchX, val_batchY in val_dataset:\n", | |
" # Define the encoder and decoder inputs, and the decoder output\n", | |
" encoder_input = val_batchX[:, 1:]\n", | |
" decoder_input = val_batchY[:, :-1]\n", | |
" decoder_output = val_batchY[:, 1:]\n", | |
"\n", | |
" # Generate a prediction\n", | |
" prediction = training_model(encoder_input, decoder_input, training=False)\n", | |
"\n", | |
" # Compute the validation loss\n", | |
" loss = loss_fcn(decoder_output, prediction)\n", | |
" accuracy = accuracy_fcn(decoder_output, prediction)\n", | |
" val_loss(loss)\n", | |
" val_accuracy(accuracy)\n", | |
" val_loss_dict[epoch+1] = val_loss.result()\n", | |
" val_accuracy_dict[epoch+1] = val_accuracy.result()\n", | |
"\n", | |
" # Print epoch number and accuracy and loss values at the end of every epoch\n", | |
" print(\"Epoch %d: Training Accuracy %.4f, Validation Accuracy %.4f\" % (epoch+1, train_accuracy.result(), val_accuracy.result()))\n", | |
" else:\n", | |
" print(\"Epoch %d: Training Accuracy %.4f\" % (epoch+1, train_accuracy.result()))\n", | |
"\n", | |
" train_loss_dict[epoch+1] = train_loss.result()\n", | |
" train_accuracy_dict[epoch+1] = train_accuracy.result()\n", | |
"\n", | |
" # Save a checkpoint after each epoch\n", | |
" if (epoch+1) % 1 == 0:\n", | |
" save_path = ckpt_manager.save()\n", | |
" training_model.save_weights(f\"model_weights/epoch{epoch+1}.ckpt\")\n", | |
" print(\"Checkpoint saved at %s\" % (save_path))\n", | |
" print(\"Weights saved at %s\" % (f\"model_weights/epoch{epoch+1}.ckpt\"))\n", | |
" stdout.flush()\n", | |
"\n", | |
"except KeyboardInterrupt:\n", | |
" print(\"Training stopped\")\n", | |
"\n", | |
"print(\"Total time taken: %.2f min\" % ((float(time()-start_time))/60.0))\n", | |
"print()\n", | |
"training_model.summary()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 12, | |
"id": "6962bb52-878e-4b61-9c18-4bf0ac92c13e", | |
"metadata": { | |
"colab": { | |
"base_uri": "https://localhost:8080/", | |
"height": 487 | |
}, | |
"collapsed": true, | |
"executionInfo": { | |
"elapsed": 1681, | |
"status": "ok", | |
"timestamp": 1719242238389, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "6962bb52-878e-4b61-9c18-4bf0ac92c13e", | |
"jupyter": { | |
"outputs_hidden": true, | |
"source_hidden": true | |
}, | |
"outputId": "c27e7d5e-d975-44b7-a9fa-34ff042ba9c4", | |
"tags": [] | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"image/png": "iVBORw0KGgoAAAANSUhEUgAABHcAAAHWCAYAAAD912X3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAADSOUlEQVR4nOzdd3hT5fvH8XfSpnswWkpbyip7lr2XLBWQKYIo4FYEUX5+VdzgwIkIqLgAUTayFATKnrL3powyC2UVKF3J+f0RLWIBgSZNx+d1Xb1Mnpxz7vs8rTS98wyTYRgGIiIiIiIiIiKSI5ldnYCIiIiIiIiIiNw9FXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdERERERERERHIwFXdEJNdZunQpJpOJpUuXujoVERERERERp1NxRyQXGzt2LCaTiQ0bNrg6lZuqUqUKRYsWxTCMmx7ToEEDQkJCSEtLy8LMRERERG7P119/jclkok6dOq5ORUTyKBV3RMSlevTowdGjR1mxYsUNXz98+DBr1qzhoYcewt3dPYuzExEREflv48ePp3jx4qxbt44DBw64Oh0RyYNU3BERl3r44YcxmUxMmDDhhq9PnDgRwzDo0aNHFmcmIiIi8t8OHTrE6tWrGTp0KMHBwYwfP97VKd3QlStXXJ2CiDiRijsiwubNm7nvvvsICAjAz8+P5s2b8+eff153TGpqKoMGDaJ06dJ4eXlRsGBBGjZsSHR0dPoxp06d4rHHHqNIkSJ4enoSGhpK+/btOXz48E1jR0RE0LhxY6ZNm0ZqamqG1ydMmEBkZCR16tThyJEj9OnTh7Jly+Lt7U3BggV58MEHb3n9vxUvXpzevXtnaG/atClNmza9ri05OZl33nmHUqVK4enpSUREBK+88grJycnXHRcdHU3Dhg3Jly8ffn5+lC1bltdff/0/cxEREZHcY/z48eTPn582bdrQpUuXGxZ3Lly4wEsvvUTx4sXx9PSkSJEi9OzZk/j4+PRjkpKSePfddylTpgxeXl6EhobSqVMnYmJigJuvKXj48GFMJhNjx45Nb+vduzd+fn7ExMRw//334+/vn/5B2YoVK3jwwQcpWrRo+vucl156iatXr2bIe8+ePXTt2pXg4GC8vb0pW7Ysb7zxBgBLlizBZDIxY8aMDOdNmDABk8nEmjVr7rg/ReTuaI6DSB63c+dOGjVqREBAAK+88goWi4Vvv/2Wpk2bsmzZsvS54++++y5DhgzhySefpHbt2iQkJLBhwwY2bdpEy5YtAejcuTM7d+6kX79+FC9enNOnTxMdHU1sbCzFixe/aQ49evTg6aefZv78+bRt2za9ffv27ezYsYO3334bgPXr17N69Wq6detGkSJFOHz4MN988w1NmzZl165d+Pj4ZLo/bDYbDzzwACtXruTpp5+mfPnybN++nS+++IJ9+/Yxc+bM9H5r27YtVapUYfDgwXh6enLgwAFWrVqV6RxEREQk5xg/fjydOnXCw8OD7t27880337B+/Xpq1aoFwOXLl2nUqBG7d+/m8ccfp3r16sTHxzN79myOHTtGUFAQVquVtm3bsmjRIrp160b//v25dOkS0dHR7Nixg8jIyDvOKy0tjdatW9OwYUM+++yz9PdJU6dOJTExkeeee46CBQuybt06RowYwbFjx5g6dWr6+du2baNRo0ZYLBaefvppihcvTkxMDL/99hsffPABTZs2JSIigvHjx9OxY8cMfRIZGUm9evUy0bMickcMEcm1xowZYwDG+vXrb3pMhw4dDA8PDyMmJia97cSJE4a/v7/RuHHj9LaqVasabdq0uel1zp8/bwDGp59+esd5njt3zvD09DS6d+9+Xftrr71mAMbevXsNwzCMxMTEDOeuWbPGAIxx48alty1ZssQAjCVLlqS3FStWzOjVq1eG85s0aWI0adIk/fnPP/9smM1mY8WKFdcdN2rUKAMwVq1aZRiGYXzxxRcGYJw5c+ZOb1dERERyiQ0bNhiAER0dbRiGYdhsNqNIkSJG//790495++23DcCYPn16hvNtNpthGIYxevRoAzCGDh1602Nu9P7GMAzj0KFDBmCMGTMmva1Xr14GYLz22msZrnej91NDhgwxTCaTceTIkfS2xo0bG/7+/te1/TMfwzCMgQMHGp6ensaFCxfS206fPm24u7sb77zzToY4IuI8mpYlkodZrVYWLFhAhw4dKFmyZHp7aGgoDz/8MCtXriQhIQGAfPnysXPnTvbv33/Da3l7e+Ph4cHSpUs5f/78HeWRP39+7r//fmbPnp0+H9wwDCZNmkTNmjUpU6ZMeoy/paamcvbsWUqVKkW+fPnYtGnTHcW8malTp1K+fHnKlStHfHx8+tc999wD2Icgg70/AGbNmoXNZnNIbBEREclZxo8fT0hICM2aNQPAZDLx0EMPMWnSJKxWKwC//vorVatWzTC65e/j/z4mKCiIfv363fSYu/Hcc89laPvn+6krV64QHx9P/fr1MQyDzZs3A3DmzBmWL1/O448/TtGiRW+aT8+ePUlOTmbatGnpbZMnTyYtLY1HHnnkrvMWkTun4o5IHnbmzBkSExMpW7ZshtfKly+PzWbj6NGjAAwePJgLFy5QpkwZKleuzP/+9z+2bduWfrynpycff/wxf/zxByEhITRu3JhPPvmEU6dO3VYuPXr04MqVK8yaNQuA1atXc/jw4esWUr569Spvv/02EREReHp6EhQURHBwMBcuXODixYuZ6Yp0+/fvZ+fOnQQHB1/39XeB6fTp0wA89NBDNGjQgCeffJKQkBC6devGlClTVOgRERHJI6xWK5MmTaJZs2YcOnSIAwcOcODAAerUqUNcXByLFi0CICYmhkqVKt3yWjExMZQtW9ahO4O6u7tTpEiRDO2xsbH07t2bAgUK4OfnR3BwME2aNAFIfz918OBBgP/Mu1y5ctSqVeu6dYbGjx9P3bp1KVWqlKNuRURug9bcEZHb0rhxY2JiYpg1axYLFizghx9+4IsvvmDUqFE8+eSTALz44ou0a9eOmTNnMn/+fN566y2GDBnC4sWLqVat2i2v37ZtWwIDA5kwYQIPP/wwEyZMwM3NjW7duqUf069fP8aMGcOLL75IvXr1CAwMxGQy0a1bt/8sqtzsUy+r1Yqbm1v6c5vNRuXKlRk6dOgNj4+IiADsn3otX76cJUuWMGfOHObNm8fkyZO55557WLBgwXXXFBERkdxn8eLFnDx5kkmTJjFp0qQMr48fP55WrVo5LN6t3svciKenJ2azOcOxLVu25Ny5c7z66quUK1cOX19fjh8/Tu/eve/qQ6qePXvSv39/jh07RnJyMn/++ScjR4684+uISOaouCOShwUHB+Pj48PevXszvLZnzx7MZnN6MQOgQIECPPbYYzz22GNcvnyZxo0b8+6776YXdwAiIyP5v//7P/7v//6P/fv3ExUVxeeff84vv/xyy1w8PT3p0qUL48aNIy4ujqlTp3LPPfdQuHDh9GOmTZtGr169+Pzzz9PbkpKSuHDhwn/ea/78+W943JEjR66bkhYZGcnWrVtp3rz5fw6DNpvNNG/enObNmzN06FA+/PBD3njjDZYsWUKLFi3+MycRERHJucaPH0+hQoX46quvMrw2ffp0ZsyYwahRo4iMjGTHjh23vFZkZCRr164lNTUVi8Vyw2Py588PkOH9zJEjR2475+3bt7Nv3z5++uknevbsmd7+z91PgfT3Rv+VN0C3bt0YMGAAEydO5OrVq1gsFh566KHbzklEHEPTskTyMDc3N1q1asWsWbOu2048Li6OCRMm0LBhQwICAgA4e/bsdef6+flRqlSp9O3BExMTSUpKuu6YyMhI/P39M2whfjM9evQgNTWVZ555hjNnzlw3JevvfA3DuK5txIgRN/3E6t+5/Pnnn6SkpKS3/f777+nTzv7WtWtXjh8/zvfff5/hGlevXk1fE+jcuXMZXo+KigK47fsVERGRnOnq1atMnz6dtm3b0qVLlwxfffv25dKlS8yePZvOnTuzdevWG24Z/vf7ms6dOxMfH3/DES9/H1OsWDHc3NxYvnz5da9//fXXt5333yOL//l+yjAMvvzyy+uOCw4OpnHjxowePZrY2Ngb5vO3oKAg7rvvPn755RfGjx/PvffeS1BQ0G3nJCKOoZE7InnA6NGjmTdvXob2/v378/777xMdHU3Dhg3p06cP7u7ufPvttyQnJ/PJJ5+kH1uhQgWaNm1KjRo1KFCgABs2bGDatGn07dsXgH379tG8eXO6du1KhQoVcHd3Z8aMGcTFxV03tepWmjRpQpEiRZg1axbe3t506tTputfbtm3Lzz//TGBgIBUqVGDNmjUsXLiQggUL/ue1n3zySaZNm8a9995L165diYmJ4Zdffsmwteijjz7KlClTePbZZ1myZAkNGjTAarWyZ88epkyZwvz586lZsyaDBw9m+fLltGnThmLFinH69Gm+/vprihQpQsOGDW/rfkVERCRnmj17NpcuXeKBBx644et169YlODiY8ePHM2HCBKZNm8aDDz7I448/To0aNTh37hyzZ89m1KhRVK1alZ49ezJu3DgGDBjAunXraNSoEVeuXGHhwoX06dOH9u3bExgYyIMPPsiIESMwmUxERkby+++/p68HeDvKlStHZGQkL7/8MsePHycgIIBff/31hpthDB8+nIYNG1K9enWefvppSpQoweHDh5kzZw5btmy57tiePXvSpUsXAN57773b70gRcRzXbdQlIs7291boN/s6evSoYRiGsWnTJqN169aGn5+f4ePjYzRr1sxYvXr1ddd6//33jdq1axv58uUzvL29jXLlyhkffPCBkZKSYhiGYcTHxxvPP/+8Ua5cOcPX19cIDAw06tSpY0yZMuWOcv7f//5nAEbXrl0zvHb+/HnjscceM4KCggw/Pz+jdevWxp49ezJsc36zrUI///xzIzw83PD09DQaNGhgbNiwIcNW6IZhGCkpKcbHH39sVKxY0fD09DTy589v1KhRwxg0aJBx8eJFwzAMY9GiRUb79u2NsLAww8PDwwgLCzO6d+9u7Nu3747uV0RERHKedu3aGV5eXsaVK1duekzv3r0Ni8VixMfHG2fPnjX69u1rhIeHGx4eHkaRIkWMXr16GfHx8enHJyYmGm+88YZRokQJw2KxGIULFza6dOlixMTEpB9z5swZo3PnzoaPj4+RP39+45lnnjF27Nhxw63QfX19b5jXrl27jBYtWhh+fn5GUFCQ8dRTTxlbt27NcA3DMIwdO3YYHTt2NPLly2d4eXkZZcuWNd56660M10xOTjby589vBAYGGlevXr3NXhQRRzIZxr/G1YmIiIiIiIjcprS0NMLCwmjXrh0//vijq9MRyZO05o6IiIiIiIjctZkzZ3LmzJnrFmkWkaylkTsiIiIiIiJyx9auXcu2bdt47733CAoKYtOmTa5OSSTP0sgdERERERERuWPffPMNzz33HIUKFWLcuHGuTkckT9PIHRERERERERGRHEwjd0REREREREREcjAVd0REREREREREcjB3VyeQGTabjRMnTuDv74/JZHJ1OiIiInIThmFw6dIlwsLCMJv12ZIr6f2TiIhIznAn759ydHHnxIkTREREuDoNERERuU1Hjx6lSJEirk4jT9P7JxERkZzldt4/5ejijr+/P2C/0YCAABdnk7VSU1NZsGABrVq1wmKxuDqdHEv9mHnqQ8dQP2ae+tAxnNWPCQkJREREpP/uFtfR+yf9O5FZ6sfMUx86hvox89SHjpEd3j/l6OLO30OJAwIC8uSbEx8fHwICAvQ/YSaoHzNPfegY6sfMUx86hrP7UdOAXE/vn/TvRGapHzNPfegY6sfMUx86RnZ4/6RJ7yIiIiIiIiIiOZiKOyIiIiIiIiIiOZiKOyIiIiIiIiIiOViOXnNHRERyNqvVSmpqqkOulZqairu7O0lJSVitVodcMy/KTD9aLBbc3NyclJmIiIiI3IyKOyIi4hKXL1/m2LFjGIbhkOsZhkHhwoU5evSoFu3NhMz0o8lkokiRIvj5+TkpOxERERG5ERV3REQky1mtVo4dO4aPjw/BwcEOKcbYbDYuX76Mn58fZrNmHd+tu+1HwzA4c+YMx44do3Tp0hrBIyIiIpKFVNwREZEsl5qaimEYBAcH4+3t7ZBr2mw2UlJS8PLyUnEnEzLTj8HBwRw+fJjU1FQVd0RERESykN79ioiIy2j6VO6i76eIiIiIa6i4IyIiIiIiIiKSg6m4IyIiIiIiIiKSg6m4IyIi4kLFixdn2LBht3380qVLMZlMXLhwwWk5iYiIiEjOouKOiIjIbTCZTLf8evfdd+/quuvXr+fpp5++7ePr16/PyZMnCQwMvKt4IiIiIpL7aLesmzAMgzSbgcVN9S8REYGTJ0+mP548eTJvv/02e/fuTW/z8/NLf2wYBlarFXf3//41GxwcfEd5eHh4ULhw4Ts6R0REREScx2YzSLG6NgdVLm5g0rpYmn62lInrYl2diohInmAYBokpaZn+uppiveNzDMO4rRwLFy6c/hUYGIjJZEp/vmfPHvz9/fnjjz+oUaMGnp6erFy5kpiYGNq3b09ISAh+fn7UqlWLhQsXXnfdf0/LMplM/PDDD3Ts2BEfHx9Kly7N7Nmz01//97SssWPHki9fPubPn0/58uXx8/Pj3nvvva4YlZaWxgsvvEC+fPkoWLAgr776Kr169aJDhw53/T0TERERyetsNoM5207ywNdr+OOYa8srGrlzA5eT0zhyNpGZm4/Ts15xV6cjIpLrXU21UuHt+S6JvWtwa3w8HPPr8LXXXuOzzz6jZMmS5M+fn6NHj3L//ffzwQcf4Onpybhx42jXrh179+6laNGiN73OoEGD+OSTT/j0008ZMWIEPXr04MiRIxQoUOCGxycmJvLZZ5/x888/YzabeeSRR3j55ZcZP348AB9//DHjx49nzJgxlC9fni+//JKZM2fSrFkzh9y3iIiISF5itRnM2X6SEYv2s//0ZQBOWUykpNmwWFyTk0bu3MADVcMwm2BT7AVizya6Oh0REckhBg8eTMuWLYmMjKRAgQJUrVqVZ555hkqVKlG6dGnee+89IiMjrxuJcyO9e/eme/fulCpVig8//JDLly+zbt26mx6fmprKqFGjqFmzJtWrV6dv374sWrQo/fURI0YwcOBAOnbsSLly5Rg5ciT58uVz1G2LiIiI5AlWm8HMzcdp9cUyXpi4mf2nL+Pv5U6/ZiV5LcqKh7vrSiwauXMDhQK8qB8ZxMoD8czacpx+zUu7OiURkVzN2+LGrsGtM3UNm83GpYRL+Af4Yzbf/i9Wb4tbpuL+U82aNa97fvnyZd59913mzJnDyZMnSUtL4+rVq8TG3nrab5UqVdIf+/r6EhAQwOnTp296vI+PD5GRkenPQ0ND04+/ePEicXFx1K5dO/11Nzc3atSogc1mu6P7ExEREcmL0qw2Zm05wVdLDnAw/goAgd4WnmhYgt4NiuPtBnPn7nNpjiru3ET7qDBWHohn5pbj9L2nFCaTydUpiYjkWiaTKdNTo2w2G2kebvh4uN9RcceRfH19r3v+8ssvEx0dzWeffUapUqXw9vamS5cupKSk3PI6ln+N5zWZTLcsxNzo+NtdS0hEREREbizVamPG5uN8teQAR/6a1ZPPx8JTjUrSs14x/L3s78FSU1NdmSag4s5N3VupMG/O3EHMmSvsPJFApXBtOSsiIndm1apV9O7dm44dOwL2kTyHDx/O0hwCAwMJCQlh/fr1NG7cGACr1cqmTZuIiorK0lxEREREcoKUNBvTNx3jq6UHOHruKgAFfD14qlFJHq1XDD/P7FdKyX4ZZRP+XhZalA9hzvaTzNx8XMUdERG5Y6VLl2b69Om0a9cOk8nEW2+95ZKpUP369WPIkCGUKlWKcuXKMWLECM6fP69RqSIiIiL/kJJmY+rGo3y9JIbjF+xFnSA/D55uXJJH6hZz2CYczqAFlW+hfVQYALO3nsBq0/B2ERG5M0OHDiV//vzUr1+fdu3a0bp1a6pXr57lebz66qt0796dnj17Uq9ePfz8/GjdujVeXl5Znotk9NVXX1G8eHG8vLyoU6fOLRfPBrhw4QLPP/88oaGheHp6UqZMGebOnZtF2YqIiOQ+yWlWfv7zCE0/XcIbM3Zw/MJVgv09ebNNeVa8cg9PN47M1oUd0MidW2pathCB3hZOX0rmz4NnaVAqyNUpiYhINtC7d2969+6d/rxp06Y3XOOmePHiLF68+Lq2559//rrn/56mdaPrXLhw4aax/p0LQIcOHa47xt3dnREjRjBixAjAvj5R+fLl6dq16w3vT7LO5MmTGTBgAKNGjaJOnToMGzaM1q1bs3fvXgoVKpTh+JSUFFq2bEmhQoWYNm0a4eHhHDlyRLufiYiI3IWkVCuT1x/lm6UxnEpIAqCQvyfPNY2ke+2ieDlw4w1nU3HnFjzczdxfOZSJ62KZufm4ijsiIpIjHTlyhAULFtCkSROSk5MZOXIkhw4d4uGHH3Z1anne0KFDeeqpp3jssccAGDVqFHPmzGH06NG89tprGY4fPXo0586dY/Xq1ekLaRcvXjwrUxYREcnxklKtTFgby6hlMZy+lAxA4QAv+jSLpGvNiBxV1Pmbijv/oUNUGBPXxTJvxyne61ApR36TRUQkbzObzYwdO5aXX34ZwzCoVKkSCxcupHz58q5OLU9LSUlh48aNDBw4ML3NbDbTokUL1qxZc8NzZs+eTb169Xj++eeZNWsWwcHBPPzww7z66qu4ud34PUpycjLJycnpzxMSEgD7zh7ZYXePrPT3/ea1+3Y09WPmqQ8dQ/2YeXmtD6+mWJm4/ijfrzxM/GX77qWhgV4807gEXaqH4+luBmykpt7ZGonO6sc7uZ6KO/+hVvEChAV6ceJiEov3nOb+yqGuTklEROSOREREsGrVKlenIf8SHx+P1WolJCTkuvaQkBD27Nlzw3MOHjzI4sWL6dGjB3PnzuXAgQP06dOH1NRU3nnnnRueM2TIEAYNGpShfcGCBfj4+GT+RnKg6OhoV6eQK6gfM0996Bjqx8zL7X2YbIWVp0wsPmnmcqp9Q4kCngYtw23UDr6Me/x2Fi3Ynuk4ju7HxMTE2z5WxZ3/YDabeCAqnFHLYpi5+biKOyIiIuIyNpuNQoUK8d133+Hm5kaNGjU4fvw4n3766U2LOwMHDmTAgAHpzxMSEoiIiKBVq1YEBARkVerZQmpqKtHR0bRs2TJ9WpvcOfVj5qkPHUP9mHm5vQ8vJ6cxfu1Rflx1mPOJ9lEwRfJ706dJCTpEhWFxc8weU87qx79H294OFXduQ4dqYYxaFsPSvWe4mJhKoE/u+6EXERGRrBUUFISbmxtxcXHXtcfFxVG4cOEbnhMaGorFYrluClb58uU5deoUKSkpeHh4ZDjH09MTT0/PDO0WiyVXvpG/HXn53h1J/Zh56kPHUD9mXm7rw0tJqfy0+jA/rDzEhb+KOsUL+vB8s1J0qBbusKLOvzm6H+/kWtoK/TaUKxxAucL+pFhtzN1x0tXpiIiISC7g4eFBjRo1WLRoUXqbzWZj0aJF1KtX74bnNGjQgAMHDmCzXVsLYN++fYSGht6wsCMiIpKXXLyaypcL99Pgo8V8tmAfFxJTKRnky9CuVVk4oAkP1oxwWmHH1XLnXTlB+6hwAGZuPu7iTERERCS3GDBgAN9//z0//fQTu3fv5rnnnuPKlSvpu2f17NnzugWXn3vuOc6dO0f//v3Zt28fc+bM4cMPP+T555931S2IiIi43MXEVIZG76Phx4v5YuE+EpLSiAz25ctuUUQPaEKn6kVwz6VFnb9pWtZteiAqjI/n7WHtoXOcuHCVsHzerk5JREREcriHHnqIM2fO8Pbbb3Pq1CmioqKYN29e+iLLsbGxmM3X3oxGREQwf/58XnrpJapUqUJ4eDj9+/fn1VdfddUtiIiIuMz5KymMXnWIsasOcyk5DYAyIX70u6c091cOxc1scnGGWUfFndsUns+b2iUKsO7QOWZvPcGzTSJdnZKIiIjkAn379qVv3743fG3p0qUZ2urVq8eff/7p5KxERESyr3NXUvhhxUF+Wn2YKylWAMoV9ueF5qW5t2JhzHmoqPO33D0uycE6aGqWiIhkQtOmTXnxxRfTnxcvXpxhw4bd8hyTycTMmTMzHdtR1xERERFxlfjLyQz5YzcNP17M10tjuJJipUJoAKMeqcHcFxpxf+XQPFnYAY3cuSP3Vy7MO7N3sOfUJfacSqBc4by1faiISF7Wrl07UlNTmTdvXobXVqxYQePGjdm6dStVqlS57WuuX78eX19fR6bJu+++y8yZM9myZct17SdPniR//vwOjSUiIiKSFU5fSuL75Qf55c9YrqbaR+pUCg+gf/MytChfCJMpbxZ0/knFnTuQz8eDpmULEb0rjpmbT/DafSruiIjkFU888QSdO3fm2LFjFClS5LrXxowZQ82aNe+osAMQHBzsyBRv6WZba4uIiIhkVycvXuW75QeZsDaW5DT7TpFViwTSv0VpmpVVUeefNC3rDv09NWv2luPYbIaLsxERySUMA1KuZP4rNfHOzzFu79/ytm3bEhwczNixY69rv3z5MlOnTqVDhw50796d8PBwfHx8qFy5MhMnTrzlNf89LWv//v00btwYLy8vKlSoQHR0dIZzXn31VcqUKYOPjw8lS5bkrbfeIjU1FYCxY8cyaNAgtm7dislkwmQypef772lZ27dv55577sHb25uCBQvy9NNPc/ny5fTXH3vsMTp06MBnn31GaGgoBQsW5Pnnn0+PJSIiIuIsMWcu87+pW2n8yRLGrDpMcpqNakXzMfaxWsx8vgH3lAtRYedfNHLnDjUvXwg/T3dOXExi/eFz1ClZ0NUpiYjkfKmJ8GFYpi5hBvLdzYmvnwCP/54a5e7uTs+ePRk7dixvvPFG+huKqVOnYrVaeeSRR5g6dSqvvvoqAQEBzJkzh0cffZTIyEhq1679n9e32Wx06tSJkJAQ1q5dy8WLF69bn+dv/v7+jB07lrCwMLZv385TTz2Fv78/r7zyCg899BA7duxg3rx5LFy4EIDAwMAM17hy5QqtW7emXr16rF+/ntOnT/Pkk0/St29fRo8enX7ckiVLCA0NZcmSJRw4cICHHnqIqKgonnrqqf+8HxEREZE7tf3YRb5eeoB5O0+lf/5Wt2QB+jQtRaPSQSro3IKKO3fIy+LGvZUKM23jMWZuOaHijohIHvL444/z6aefsmzZMpo2bQrYp2R17tyZYsWK8fLLL6cf269fP+bPn8+UKVNuq7izcOFC9uzZw/z58wkLsxe6PvzwQ+67777rjnvzzTfTHxcvXpyXX36ZSZMm8corr+Dt7Y2fnx/u7u63nIY1YcIEkpKSGDduXPqaPyNHjqRdu3YMGTIEb29vAPLnz8/IkSNxc3OjXLlytGnThkWLFqm4IyIiIg5jGAZrYs7y9dIYVh6IT29vUT6EPs0iqV5UawbeDhV37kKHqHCmbTzG3O0nGfRARTzcNbtNRCRTLD72ETSZYLPZSLh0iQB/f8zmO/h32eJz24eWK1eO+vXrM3r0aJo2bcqBAwdYsWIFgwcPxmq18uGHHzJlyhSOHz9OSkoKycnJ+Pjc3vV3795NREREemEH7Fte/9vkyZMZPnw4MTExXL58mbS0NAIC7mwNuN27d1O1atXrFnNu0KABNpuNvXv3EhUVBUDFihVxc3NLPyY0NJTt27ffUSwRERGRG7HZDBbsiuObZTFsPXoBADezifZVw3i2aSRlQvxdm2AOo+LOXagXWZBC/p6cvpTM0r2naVVRi1SKiGSKyXRbU6NuyWYDi9V+nTsp7tyhJ554gn79+vHVV18xZswYIiMjadKkCR9//DFffvklw4YNo3Llyvj6+vLiiy+SkpLisNhr1qyhR48eDBo0iNatWxMYGMikSZP4/PPPHRbjnywWy3XPTSYTNpvNKbFEREQkb0i12pi15QSjlsVw4LR9vT9PdzPdakXwZKOSRBS4/Q/e5BoVd+6Cm9lEu6ph/LjyELO2nFBxR0QkD+natSv9+/dnwoQJjBs3jueeew6TycSqVato3749jzzyCGAfSbRv3z4qVKhwW9ctX748R48e5eTJk4SGhgLw559/XnfM6tWrKVasGG+88UZ625EjR647xsPDA6vV+p+xxo4dy5UrV9JH76xatQqz2UzZsmVvK18RERGRO3E1xcqk9bF8v/wgJy4mAeDv5U7PesV4rEEJgvw8XZxhzqb5RHfp712zFu6O41KSdg4REckr/Pz8eOihhxg4cCAnT56kd+/eAJQuXZro6GhWr17N7t27eeaZZ4iLi7vt67Zo0YIyZcrQq1cvtm7dyooVK64r4vwdIzY2lkmTJhETE8Pw4cOZMWPGdccUL16cQ4cOsWXLFuLj40lOTs4Qq0ePHnh5edGrVy927NjBkiVL6NevH48++ighISF33ikiIiIiN3ExMZURi/bT4OPFDPptFycuJhHk58mr95Zj9Wv38L/W5VTYcQAVd+5SpfAASgb7kpxmY96OU65OR0REstATTzzB+fPnad26dfoaOW+++SbVq1endevWNG3alMKFC9OhQ4fbvqbZbGbGjBlcvXqV2rVr8+STT/LBBx9cd8wDDzzASy+9RN++fYmKimL16tW89dZb1x3TuXNn7r33Xpo1a0ZwcPANt2P38fFh/vz5nDt3jlq1atGlSxeaN2/OyJEj77wzRERERG7gdEISH87dTf2PFvF59D7OXUkhooA373eoxMpXm/Fc00j8vSz/fSG5LZqWdZdMJhMdosIZGr2PWVtO8GDNCFenJCIiWaRevXoYf+/P+ZcCBQowc+bMW563dOnS654fPnz4uudlypRhxYoV17X9O84nn3zCJ598cl3bP7dM9/T0ZNq0aRli//s6lStXZvHixRmO+3tNnTFjxmRYmHrYsGEZjhcRERH5p8PxV/h2+UF+3XiMFKv9fUW5wv481zSSNpVDcXfTGBNnUHEnE9pHhTE0eh+rY+I5nZBEoQAvV6ckIiIiIiIikuV2nrjIN0tjmLv9JLa/PlOqWSw/fZpF0qxsIUwmk2sTzOVU3MmEYgV9qVY0H5tjLzB76wmebFTS1SmJiIiIiIiIZJl1h87x9dIDLN17Jr2tWdlgnmtaitolCrgws7xFxZ1M6hAVzubYC8zaouKOiIiIiIiI5H6GYbB4z2m+WRrDhiPnATCboE2VMJ5rEkmFsAAXZ5j3qLiTSW2qhDL4911sP36RmDOXiQz2c3VKIiIiIiIiIg6XZrUxZ/tJvlkaw55TlwDwcDPTpWYRnmlckmIFfV2coYtcPo1vkms3WlJxJ5OC/DxpVDqIpXvPMGvzcQa0KuvqlEREcox/L/IrOZu+nyIiIrlTUqqVqRuP8d3yGI6euwqAr4cbj9QtxhMNS+Sd9WcNA84fgpPb4NR2OLUNTm7DcvkUlf2rAI+7LDUVdxygQ1Q4S/eeYeaWE7zUsowWihIR+Q9ubm4ApKSk4O3t7eJsxFFSUlKAa99fERERydkSklL55c8jjF55mPjLyQAU8PXg8QbFebRucQJ9cvFW5tZUOLM3vYDDqb8KOskJGQ41MOFmpLggyWtU3HGAlhVC8La4EXsukc1HL1C9aH5XpyQikq25u7vj4+PDmTNnsFgsGbbcvhs2m42UlBSSkpIccr286m770WazcebMGXx8fHB319sLERGRnCz+cjKjVx7i5z+PcCkpDYDwfN481agED9UqirdHLvsgJ/kyxO38q5Cz1f7f07vBeoOCjZsnhFSAwpWhcBUIrUpagTKsWriM+7M+83R69+UAvp7utKoYwqwtJ5i1+biKOyIi/8FkMhEaGsqhQ4c4cuSIQ65pGAZXr17F29tbIygzITP9aDabKVq0qPpfREQkhzp6LpHvVxxk8vqjJKfZAChVyI/nmkTyQFQYFrdc8AHalfhrBZxT2+2jcs4eAG4wvdwz0F7ECa3yVyGnCgSVAbd/jVhKTc2S1G9FxR0H6RAVzqwtJ/h920nebFshd/zQi4g4kYeHB6VLl06fypNZqampLF++nMaNG2Ox5OIhwk6WmX708PDQqCkREZEcaF/cJX5YFcvsrSew2uxFjqoR+ejTNJKW5UMwm3PgBzeGARdir59WdXIbXDpx4+P9Q+0FnH8Wc/IXhxzyoZWKOw7SsHQQBXw9OHslhZUH4mlWtpCrUxIRyfbMZjNeXo5ZgM/NzY20tDS8vLxU3MkE9aOIiEjeYBgGG46c5/s9ZnasWZPe3qh0EM81jaReyYI5ZzSuNQ3i9/1rfZxtkHTxxscXiLx+NE7hKuCXs/+GV3HHQSxuZtpWCWXcmiPM2nxcxR0RERERERHJdq6mWJm15Tg/rTnC7pMJgBmTCe6rVJhnm0RSpUg+V6d4aymJf62Ps/XatKrTuyAtKeOxZgsUKv9XAafqX+vkVAJP/6zP28lU3HGg9lHhjFtzhAW74khMScPHQ90rIiIiIiIirhd7NpFf1h5h8vqjXLxqXyPGy2KmWv403u3WiLJh+Vyb4I3YrHBsAxxde21Uztn9YNgyHuvh949Fjv8ajRNcDtw9sj5vF1D1wYGqF81H0QI+xJ5LJHpXHO2jwl2dkoiIiIiIiORRNpvBygPxjFtzmEV7TmP8tWZwRAFvHq1bjI5VQ1m9NJqSwb6uTfSf0pLh4DLY8zvsnQtXzmQ8xrdQxmlV+UtAHl77T8UdBzKZTLSPCmPE4gPM3HxcxR0RERERERHJcpeSUvl14zHGrTnCwfgr6e2NSgfRu35xmpYthJvZRGo22OUJgKQE2L/AXtDZHw0pl6+95hUIJRpDaFX71KrQKuBf2HW5ZlMq7jhY+6hwRiw+wPL98Zy9nExBP09XpyQiIiIiIiJ5wP64S4xbc4Tpm45xJcUKgJ+nO11qFOHResWIDPZzcYb/cCnOPjJnz+/2kTq2fxSa/EOhXBso1xaKN8y49bhkoOKOg5Uq5Eel8AB2HE9gzvaT9KxX3NUpiYiIiIiISC6VZrWxaM9pflp9mNUxZ9PbSxfyo2f94nSsFo6fZzb50/9sDOyZYy/oHF0HGNdeCyrzV0GnHYRVy9NTrO5GNvkO5y4dosLZcTyBmZuPq7gjIiIiIiIiDnfuSgqT1scy/s9Yjl+4CoDZBC0rhNCrXnHqRWaDrcwNA05uvVbQOb3r+tfDa9hH55RrC8FlXJNjLqHijhO0qxrGB3N3syn2ArFnEyla0MfVKYmIiIiIiEgusP3YRX5ac5jZW0+QkmbfNSq/j4VutYvSo05RiuR38d+f1jSIXfNXQWcOXIy99prZ3T7NqlxbKHs/BGqdWkdRcccJQgK8qB9ZkFUHzjJry3H6NS/t6pREREREREQkh0pJs/HHjpP8tPowm2IvpLdXDg+kV/3itK0SipfFzXUJpl6FmCV/7XD1B1w9d+01iw+Uam6fblWmFXjnd12euZjLizvHjx/n1Vdf5Y8//iAxMZFSpUoxZswYatas6erUMqV9VDirDpxl5pbj9L2nlOuHw4mIiIiIiEiOEpeQxPi1sUxYG0v85WQALG4m2lQOpWf94lSLyOe6vzWvnod9C2DPb3BgEaQmXnvNuwCUvc8+QieyGVi8XZNjHuLS4s758+dp0KABzZo1448//iA4OJj9+/eTP3/Or+TdW6kwb87cQcyZK+w8kUCl8EBXpyQiIiIiIiLZnGEYrD98np/WHGb+jlOk2eyLDocEeNKjTjG61Y6gkL+Xa5JLOHFt/ZzDK8GWdu21wIi/1s9pA0XrgZvLx5LkKS7t7Y8//piIiAjGjBmT3laiRAkXZuQ4AV4WWpQvxNztp5i5+biKOyIiIiIiInJTV1OszNpynJ/WHGH3yYT09tolCtCrXnFaVQzB4uaCHaTO7LMXc/b8Dsc3Xv9aoQrXCjqhVUEzVlzGpcWd2bNn07p1ax588EGWLVtGeHg4ffr04amnnrrh8cnJySQnJ6c/T0iw/8CnpqaSmpqaJTnfibaVCjN3+ylmbz3Byy1L4WZ23A/63/ebHe87J1E/Zp760DHUj5mnPnQMZ/Wjvi8iIiI3Fns2kV/WHmHy+qNcvGr/fellMdOxWjiP1i1OhbCArE3IZoMTm+3TrfbMgfh9/3jRBBG1rxV0CkZmbW5yUy4t7hw8eJBvvvmGAQMG8Prrr7N+/XpeeOEFPDw86NWrV4bjhwwZwqBBgzK0L1iwAB+f7LcjVZoNvN3cOH0pmeGT51E20HB4jOjoaIdfMy9SP2ae+tAx1I+Zpz50DEf3Y2Ji4n8fJCIikkfYbAYrD8Tz0+rDLN57GuOvPxUjCnjTs25xutaMINDHknUJWVMhduVfI3TmwqUT114zW6BkU3sxp+z94B+SdXnJbXNpccdms1GzZk0+/PBDAKpVq8aOHTsYNWrUDYs7AwcOZMCAAenPExISiIiIoFWrVgQEZHE18zats+5k8objxHlF8NL9lRx23dTUVKKjo2nZsiUWSxb+T5/LqB8zT33oGOrHzFMfOoaz+vHv0bYiIiJ5WUJSKr9uPMbPa45wMP5KenuTMsH0ql+MJmUKOXTGxy2lJmHaO5/qh0fhPqwfJF289pqHH5RuaR+hU7oVeGXPv7flGpcWd0JDQ6lQocJ1beXLl+fXX3+94fGenp54enpmaLdYLNn2jXzH6hFM3nCcBTtP80FHs8O3p8vO956TqB8zT33oGOrHzFMfOoaj+1HfExERycv2x11i3JojTN90jCspVgD8Pd3pUrMIj9YtRslgv6xJJC0FDi6FndNhzxzckxOI+Ps132D7yJxybaFkE3DP+Le3ZF8uLe40aNCAvXv3Xte2b98+ihUr5qKMHK928QKEBXpx4mISi/ec5v7Koa5OSURERERERJwszWpj0Z7T/LT6MKtjzqa3ly7kR8/6xelULRxfzyz4k9yaBodX2As6u2ZD0oX0lwz/MGK8KlP8vn64F68PZscORpCs49LizksvvUT9+vX58MMP6dq1K+vWreO7777ju+++c2VaDmU2m2gXFca3yw4yc/NxFXdERERERERyKcMw2HE8gembj/Hb1hPEX04BwGyClhVC6FW/OPVKFsTk7F2lbDaIXfNXQWcWXDlz7TXfQlCxI1TqRFrhauz8Yx7FIuqqsJPDubS4U6tWLWbMmMHAgQMZPHgwJUqUYNiwYfTo0cOVaTlch6hwvl12kKV7z3AxMTVrF8YSERERERERpzp+4SozNx9nxubjHDh9Ob29oK8HD9WKoEfdYoTn83ZuEoYBxzbYCzo7Z8Clk9de8y4AFdpDpU5QrMG1Qo52s8w1XFrcAWjbti1t27Z1dRpOVT40gLIh/uyNu8TcHSfpXruoq1MSERERERGRTLiUlMof208xffMx1h46l77jlae7mZYVQuhUPZxGpYOxuJmdl4RhwMmtsONX2DkTLsZee80rEMq1g0odoUQTcNMgg9zM5cWdvKJ9tTA+mbeXmZuPq7gjIiIiIiKSA6VZbazYH8/0zcdZsPMUyWm29NfqlixAp2pFuLdyYQK8nFxIidv1V0FnOpw7eK3dw8++KHKlThB5jxZFzkNU3MkiD1S1F3fWHjrHiQtXCXP2kDwRERERERHJtJutowNQqpAfHauF06FauPOnXcXvhx3T7QWdM3uutbt7Q5nW9oJO6VZg0d+aeZGKO1mkSH4fahcvwLrD55i99QTPNol0dUoiIiIiIiJyE7daR+eBqDA6VStCpfAA5y6OfP7wtYLOqe3X2t08oFRLe0GnzL3gmUVbqUu2peJOFmpfLYx1h88xc/NxFXdERERERESymWyxjs7FY/b1c3b8Cic2XWs3u0PJZlCpM5S7376mjshfVNzJQm0qh/Lu7J3sOXWJvacuUbawv6tTEhERERERydOyxTo6l+LsW5bv+BWO/nmt3WSG4o3sBZ3y7cCngPNykBxNxZ0slM/HgyZlCrFwdxwztxzn1XvLuTolERERERGRPOdW6+hEBvvSqXoR56+jc+Us7J5tL+gcWQXG30UlExSrDxU72rcv9yvkvBwk11BxJ4t1qBbGwt1xzN5ygv+1KovZ7MT5mSIiIiIiIpLuVuvotKsaRqfq4VQOD3TeOjpXL8Ce3+3r6BxcCob12mtFakHFTlCxAwSEOSe+5Foq7mSxFuVD8PN05/iFq2w4cp7aJTSsTkRERERExFlcvo5O8iXY+4e9oBOzCKzXRgkRWvWvgk5HyF/MOfElT1BxJ4t5WdxoXbEwv246xswtx1XcERERERERcTCXraNjGHA5DuJ2wuldcHQt7I+GtKRrxxSqYC/oVOoEBbXRjjiGijsu0KFaGL9uOsbc7Sd5t11FPNyduNK6iIiIiIhIHnA76+i0jwqjSH4fxwRMugind9uLOHG7/nq8E66ez3hswVLXCjqFyjsmvsg/qLjjAvUjgwj29+TMpWSW7TtDywohrk5JREREREQkRzpx4Sq/7zjivHV00pIhft9fBZy/v3bDxaM3Pt5kthdzClWAkIpQ5l4oXBmctY6PCCruuISb2US7KmGMXnWImVuOq7gjIiIiIiJyB5LTrMzYdJwfdpqJ+XOFY9bRsdngwuHrizhxu+DsgesXPv6ngPC/ijgV7P8tVAGCyoDFK9P3KHInVNxxkQ7V7MWdhbviuJSUir+j53qKiIiIiIjkMslpVqZsOMbXSw5w8mISYC/e1ClRgE7Vw7mvcuh/r6NjGHD5tH0K1end14o5Z/ZAauKNz/EKhEIV/yrilLc/LlQevPM59P5E7paKOy5SOTyQkkG+HIy/wvydcXSpUcTVKYmIiIiIiGRLGYs6EOLvSc18ifyva1OKBwfc5MRLfxVwdl5bH+f0Lkg8e+Pj3TwhuKx9OtXfI3FCKoB/qKZVSbam4o6LmEwmOlQLZ2j0PmZtOa7ijoiIiIiIyL/csKgT4EmfpqXoHFWYRdHzCc/nDWkpcHb/X6Nw/jEi52LsjS9sMkOBktdG4fw9rapASTC7ZeEdijiGijsu1D4qjKHR+1h1IJ7Tl5Io5K95mSIiIiIiIjcq6hQO8KJPs0i61ozA68oJrBu+osah+bh/96F9XRxb2o0v5h/61yic8n+NyCkPweXA4p2FdyTiXCruuFCxgr5UK5qPzbEX+G3rSZ5oWMLVKYmIiIiIiLhMcpqVKeuP8vXSmIxFnahCeMXMg0kvQcwS3DC4bv6DZ0DGxY0LlQefAi65F5GspOKOi3WICmdz7AVmbTmu4o6IiIiIiORJtyrqPFT0Ep7bv4Xhk+DqufRzbMUasjs1jLKNOuIeVsW+c5XWxZE8SsUdF2tTJZTBv+9i27GLHDxzmZLBfq5OSUREREREJEvcrKjTv1EIXTzWYtn2AczfeO0E/zCIehiqPYLVvwgH5s6lTKmWYNHuw5K3qbjjYkF+njQqHcTSvWeYueUEA1qWcXVKIiIiIiIiTvV3UeerJTGcSvirqOPvybvVEmiR9Bvuy2dd25bc7A5l7oXqvaBU82sLHqemuih7kexHxZ1soENUOEv3nmHWluO81KI0Jg0lFBERERGRXCgp1cqUDUf5+h9FnQr+SbxXYjvVzv6Oed3+awcHlYFqj0LVbuBXyEUZi+QMKu5kAy0rhOBtcePI2US2HL1AtaL5XZ2SiIiIiIiIw/y7qOOGlc5+u+lfYA0R8Ssw7ftrpyuLD1TsBNV7QkRtraEjcptU3MkGfD3daVUxhFlbTjBrywkVd0REREREJFf4d1GnqCmOd31W0tWyHJ/kM3D6rwPDa9oLOpU6gae/S3MWyYlU3MkmOkSFM2vLCX7fdoI325TH3c3s6pRERERERETuyj+LOucTErjXvI6e3supYewAG5AM+BSEKt2g+qP2LctF5K6puJNNNCwdRAFfD+Ivp7DyQDxNy2pOqYiIiIiI5Cz/LOoUvLSHPm5L6OC1mgCugAFggsh77KN0yt4P7h6uTlkkV1BxJ5uwuJlpWyWUcWuOMGvLCRV3REREREQkx0hKtTJ5/VF+XrKFeolL+MFtKZU8D187ILAoVHvEvo15vghXpSmSa6m4k420jwpn3JojzN95isSUNHw89O0REREREZHsKynVyuR1R1i7ZBatkhfwu3k9Xhb7FuWGmwemcm3t065KNAWzlp4QcRZVD7KR6kXzUbSAD7HnEoneFUf7qHBXpyQiIiIiIpJBUqqV31ZsIH7VGO5PXUQv82lws79mK1QBc/VemKp0BZ8Crk1UJI9QcScbMZlMtI8KY8TiA8zackLFHRERERERyVaSkpJY88d4LNvG08m2CTeTAWZIcfPFreqDuNXoiTmsurYwF8liKu5kM+2jwhmx+ADL953h3JUUCvhqgTEREREREXGt5JO72T/va8KOzKIZF+2NJojLX50CDZ/Ao3JH8PB1bZIieZgmPd7M6d1gTc3ysKUK+VEpPIA0m8GcbSeyPL6IiIhkra+++orixYvj5eVFnTp1WLdu3U2PHTt2LCaT6bovLy+vLMxWRPKUlCukbhhH3LAmeH5bl0pHxlGAi5wlHztKPE7Kc+sI6b8ES41HVNgRcTGN3LmRFUNhyQdwz5vQ8KUsD98hKpwdxxOYueUEj9YrnuXxRUREJGtMnjyZAQMGMGrUKOrUqcOwYcNo3bo1e/fupVChG++cGRAQwN69e9OfmzT1QUQc7eRW0tb9iG3bNDysVwgBrIaJ1W41sEU9St3W3ajkqcKySHai4s6N+BcGWxos/QgqtIcCJbM0fLuqYXwwdzcbj5zn6LlEIgr4ZGl8ERERyRpDhw7lqaee4rHHHgNg1KhRzJkzh9GjR/Paa6/d8ByTyUThwoWzMk0RyQtSEmHndGzrf8R8YlP6H4qHbCHMs7QgpPFjtGlQHU93N5emKSI3puLOjVTtDlsnwaFl8PtL8OjMLF0QLCTAi/qRBVl14Cyzthyn7z2lsyy2iIiIZI2UlBQ2btzIwIED09vMZjMtWrRgzZo1Nz3v8uXLFCtWDJvNRvXq1fnwww+pWLHiTY9PTk4mOTk5/XlCQgIAqamppKZm/RR0V/r7fvPafTua+jHzslUfntmLefNPmLdNwpScgBlIMdz4w1aH+V73UrdpWx6tXgRPdzMYNlJTba7OOF226sccSn3oGM7qxzu5noo7N2IyQdsv4Jv6cHCpvdAT1T1LU2gfFc6qA2eZueUEzzcrpSHXIiIiuUx8fDxWq5WQkJDr2kNCQtizZ88NzylbtiyjR4+mSpUqXLx4kc8++4z69euzc+dOihQpcsNzhgwZwqBBgzK0L1iwAB+fvDk6ODo62tUp5Arqx8xzVR+abamEXthA8bOLCbp8bZrnEVshJlibs9C9ETUj/GkebOAev4NFC3a4JM/bpZ/FzFMfOoaj+zExMfG2j1Vx52YKRkKTV2HRIJj/OpRuCb5BWRb+3kqFeXPmDg6cvszOEwlUCg/MstgiIiKSPdWrV4969eqlP69fvz7ly5fn22+/5b333rvhOQMHDmTAgAHpzxMSEoiIiKBVq1YEBAQ4PefsJDU1lejoaFq2bInFYnF1OjmW+jHzXNaH5w/bR+lsnYAp8SwAaZhZZK3OeGtzYgPr8GzTSH6rGorFLfvvvaOfxcxTHzqGs/rx79G2t0PFnVup3w92/ApxO2D+G9Dp2ywLHeBloUX5QszdfopZW46ruCMiIpLLBAUF4ebmRlxc3HXtcXFxt72mjsVioVq1ahw4cOCmx3h6euLp6XnDc/PqG/m8fO+OpH7MvCzpQ2sa7JsHG0ZDzKL05jijABPSmjHZ2hTf4KL0vacU7aqE4Z4Dijr/pp/FzFMfOoaj+/FOrpXz/s/NSm4WaDccMMG2SXBg0X+e4kjto8IBmL31BFabkaWxRURExLk8PDyoUaMGixZde39hs9lYtGjRdaNzbsVqtbJ9+3ZCQ0OdlaaI5FQXj8OSITCsEkzukV7YWWlU5amUAdRP/pI/gnrxRvcWLHipCR2rFcmRhR0RsdPInf9SpAbUeQbWjrIvrtznT/DImvnpTcsGE+DlTlxCMmsPnqV+qaybFiYiIiLON2DAAHr16kXNmjWpXbs2w4YN48qVK+m7Z/Xs2ZPw8HCGDBkCwODBg6lbty6lSpXiwoULfPrppxw5coQnn3zSlbchItmFzQYHF8P60bDvDzDsix9fcc/PhJTGjEttylEjhPKhAYy8pxStKxbGbNbaniK5gYo7t+OeN2H373DhCCz7CFoOzpKwnu5utKkSysR1R5m55biKOyIiIrnMQw89xJkzZ3j77bc5deoUUVFRzJs3L32R5djYWMzma5+knz9/nqeeeopTp06RP39+atSowerVq6lQoYKrbkFEsoPLZ2DLL7BhjP1vlr8c9qvGlxcbMSepBilYqBweyNvNS9OifCFt2CKSy6i4czs8/aHNZzCxG6weCZU6Q2jVLAndPiqcieuO8sf2UwxuXwkvi1uWxBUREZGs0bdvX/r27XvD15YuXXrd8y+++IIvvvgiC7ISkWzPMODIKvtaOrtmg82+ZbLNM5C1Aa0YfLIuu+PtUzajIvLRv3lpmpYNVlFHJJdSced2lb0PKnSAXTNh9gvw1GIwO7/QUrt4AcICvThxMYkle05zX2XNqRcRERERybOunoetk+xFnfh96c3JIdWYbbmPwYfKcOmiBwC1iufnhealaVgqSEUdkVxOxZ07cd8nELMETm6Btd9CvT5OD2k2m2gXFca3yw4yc8txFXdERERERPIaw4Djm2DDj/bdfNOS7O0WXy6V6cjopCaM2O1H2l+bsNQrWZAXmpembskCKuqI5BEq7twJ/xBoNRh+6w+L34fybSFfUaeH7RAVzrfLDrJkzxkuJqYS6KMt6kREREREcr3ky7B9qn2Uzqlt19oLVeRM+R4MPRnF5E0XsNd0DBqVDqLfPaWpXaKAqzIWERdRcedOVesJWydD7Gr4fQD0mApOroaXDw2gbIg/e+Mu8ceOk3Sr7fyCkoiIiIiIuMipHfaCzrYpkHLJ3ubmCRU7EhvZnc92BvDbgpMYxgUAmpUNpl/z0lQvmt91OYuIS6m4c6fMZmj3JYxqAAeiYed0+wLLTta+WhifzNvLzC3HVdwREREREcltUq/Crlmw/kc4tu5ae4FIqPk4ewu348s18cydeAq4AkCL8iG80LwUVYrkc0nKIpJ9qLhzN4LLQKOXYemH8MerULIZ+Dh36OMDVe3FnbWHznHy4lWCfPStExERERHJ8eIPwMYxsGW8fbFkALM7lGsLNR9nu6Uqw5ccIHr2jvRT7qtUmL73lKJiWKCLkhaR7EYVgrvV8EX7YmbxeyH6bWg/0qnhiuT3oXbxAqw7fI7ZW07weH2N3hERERERyZGsqYSdX4fb+O/h8Ipr7YFFoUYvqPYom897MGLxARbvWQXYV4JoUzmUfveUpmxhfxclLiLZlYo7d8vd0z49a8y9sPlnqPIQlGjk1JDtq4Wx7vA5Zqq4IyIiIiKS81jTYNtk3Jd+RK2LsX81mqBMa6j5BJRqzobYi3w5ZT8r9scDYDZB+6hwnm8WSalCKuqIyI2puJMZxepBzcfti539/iI8uwosXk4L16ZyKO/O3snukwnsj7vstDgiIiIiIuJANhvsng1LPoD4fZiAJPdALHWexK1Wb8hXlD8PnmX4j+tZHXMWADeziY7Vwnm+WSlKBPm6NH0Ryf5U3Mms5u/Anrlw9gCs+AzuedNpofL5eNCkTCEW7o5j9raTlHdaJBERERERyTTDgAMLYfF7cHKrvc07P9b6/VkYH06rJh1YcySB4ZPWsO7wOQDczSYerFmE55qUomhBHxcmLyI5iYo7meWdD+7/BKb0hJVfQMVOEFLBaeE6VAtj4e44ftt2krLlnBZGREREREQy48hqWDQYYtfYn3v4Q73nod7zWM1ebJ8wjx+/X8fmoxftL7uZ6VqrCM82iaRIfhV1ROTOqLjjCOUfgLJtYO8c+K0/PD7fvmW6E7QoH4KfpzvHLyRx6JJTQoiIiIiIyN06sRkWv28fsQPg7gW1n4IGL4FvQQ6cvsTA6RtYf9gNuIinu5nutYvybJNICgc6b4kHEcndVNxxBJMJ7v8UDi2DY+tgw4/2f8CdwMviRuuKhfl10zE2xjungCQiIiIiInfo9B77mjq7Z9ufm92hek9o/D8ICCM5zco3C/fx9ZIYUqw2LGaDR+sW59mmpSgUoKKOiGSOijuOEhhuX3/nj//BwkFQrg0EhDklVIdqYfy66Ribz5pISbNhsTgljIiIiIiI/Jfzh2Hpx7BtEhg2wGTfSbfpa1CgBADrD59j4PTtHDht3xSlaZkgmvid4pH7ymLRm3kRcQAN/XCkWk9AkVqQcgnm/s9pYepHBhHs50FimolZW086LY6IiIiIiNzEpVMw5/9gRE3YOsFe2CnXFvqsgU7fQoESXLyayusztvPgqDUcOH2ZID9PRj5cje8eqUYBT1ffgIjkJiruOJLZDdp9aR+Cued32P2bU8K4mU30rFsUgMFzdrPnVIJT4oiIiIiIyL8knoMFb8GXUbD+B7ClQuQ98NRi6DYeCpXHMAz+2H6SlkOXMWFtLADdakWwaEAT2lYJw2QyufYeRCTXUXHH0UIqQoP+9sdz/wdJF50S5qlGJSgXaCMp1cazP28kISnVKXFERERERARISoClH8GwKrB6OKRdhYg60Ot3eHQGhNcA4MSFqzw1biPPjd/E6UvJlAzyZdLTdfmocxUCfTQFS0ScQ8UdZ2j8ChSIhEsn7evvOIGb2UTP0jbCAr04fDaR/5uyFZvNcEosEREREZE8K/UqrB4BX1aFpUPsSzAUrgwPT7XvkluiEQBWm8FPqw/TcugyFu6Ow+Jm4oV7SjG3fyPqlizo4psQkdxOxR1nsHhBu2H2xxt+hNg/nRLG1wIju1fFw81M9K44Ri2PcUocEREREZE8x5oK63+E4dVgwZtw9RwULAVdxsDTy6FMK/uuucCeUwl0/mY178zeyZUUKzWK5WfOC40Y0KosXhY3F9+IiOQFKu44S4nGEPWI/fFv/SEtxSlhKocHMqh9RQA+m7+XVQfinRJHRERERCRPsFlh62QYWRPmDLCPxg+MgPZfQZ+1UKkTmO1/RiWlWvlk3h7aDl/JlqMX8Pd0570OlZj6TD3KhPi7+EZEJC9RcceZWr0HPkFwZg+sGua0MN1qRdC1ZhFsBvSbuJkTF646LZaIiIiISK5kGPYNUb5pADOetm9x7hsM930C/TZCtUfAzT398NUH4rl32HK+XhpDms3g3oqFiR7QhEfrFsNs1oLJIpK13P/7ELlrPgXgvo/h1ydg+adQsSMElXZ4GJPJxOD2ldh5IoGdJxJ4bvwmpjxTF093DQEVEREREbklw4CDS2DRe3Bik73NKxAavAh1ngEP3+sOP38lhQ/n7mbqxmMAhAR4Mrh9JVpXLJzFiYuIXOPSkTvvvvsuJpPpuq9y5cq5MiXHq9QZSrUAa4p9epbN5pQwXhY3Rj1Sg0BvC1uPXuC933c5JY6IiIiISK4RuxZ+agc/d7QXdiy+0Ph/0H8bNBpwXWHHMAxmbTlOi6HLmLrxGCYTPFq3GNEDmqiwIyIu5/KROxUrVmThwoXpz93dXZ6SY5lM0GYofF0XjqyCLb9A9Z5OCRVRwIdh3aJ4fOx6fvkzlmoR+elco4hTYomIiIiI5Fgnt8Hi92H/fPtzNw+o9SQ0HAB+wRkOP3oukTdn7mDZvjMAlAnxY0inytQoViArsxYRuSmXV1Lc3d0pXDiXV7rzF4Nmb8CCN+wr7ZduDf4hTgnVrGwhXrinNF8u2s/rM7ZTLtSfimGBToklIiIiIpKjxO+HJR/Azhn25yY3+1o6TV6BwIwfiqZZbYxZdZih0fu4mmrFw93MC/eU4unGkXi4a/lSEck+XF7c2b9/P2FhYXh5eVGvXj2GDBlC0aJFb3hscnIyycnJ6c8TEhIASE1NJTU1NUvyvWs1nsB922RMp7Zh++MVrB1/yNTl/r7fG913n8bF2RJ7nmX743n2543MeK4ugd6WTMXLrW7Vj3J71IeOoX7MPPWhYzirH/V9ERGXuhALyz6GLRPA+GuZhEpdoNnrUDDyhqfsOH6R16ZvY8dx+98cdUoUYEinypQM9suqrEVEbptLizt16tRh7NixlC1blpMnTzJo0CAaNWrEjh078PfPuHXgkCFDGDRoUIb2BQsW4OPjkxUpZ0pgYBcan9qBeddM1iWVJC4wKtPXjI6OvmF760DY7unG0fNX6fXNIp4sa0OL9t/czfpRbp/60DHUj5mnPnQMR/djYmKiQ68nInJbLsXBis9h4xj7GpgAZe+3j6ovXOmGpySmpPFF9D5+XHkImwGB3hbeuL88D9YsgsmkN9Qikj25tLhz3333pT+uUqUKderUoVixYkyZMoUnnngiw/EDBw5kwIAB6c8TEhKIiIigVatWBAQEZEnOmWUsioM/v6JO/BTSOr8AHndX+U9NTSU6OpqWLVtisdx4VE6FWgl0/X4dO8/DEd8yPN+0ZGZSz5Vupx/l1tSHjqF+zDz1oWM4qx//Hm0rIpIlrp6HVcNh7ShI/au4XKIx3PM2RNS66WnL9p3hjRnbOXb+KgDtqobxdtsKBPt7ZkXWIiJ3zeXTsv4pX758lClThgMHDtzwdU9PTzw9M/7DarFYcs4b+XvegD2/YboQi2XFJ3DvkExd7lb3HlWsIO+3r8Qrv27jy8UHqF6sAI3LZFwgTnLYz1A2pT50DPVj5qkPHcPR/ajviYhkCZsV1v8Aiz+A5Iv2tvCa0PwtKNn0pqfFX07mvd93MWvLCfsp+bx5v0MlmpUrlAVJi4hkXrZaBezy5cvExMQQGhrq6lScx8MX2n5hf7x2FBzf6NRwXWtF0L12BIYB/Sdt5th5DYsXERERkVzozF4Ycx/88Yq9sFOoAnSbCE8uvGlhxzAMpm44Souhy5i15QRmEzzRsAQLXmqswo6I5CguLe68/PLLLFu2jMOHD7N69Wo6duyIm5sb3bt3d2VazleqBVTual/MbXZ/sDp3kcl32lWkcngg5xNT6TN+E0mpVqfGExERERHJMtZUWP4pjGoIR9eChz+0+RyeXQXl7oebrJNzOP4KPX5Yy/+mbeNCYioVQgOY+XwD3mpbAV/PbDXBQUTkP7m0uHPs2DG6d+9O2bJl6dq1KwULFuTPP/8kODgPTB1q/SF454e47bDmK6eG8rK48c0j1cnnY2HbsYsM+m2XU+OJiIiIiGSJE5vhu6aw+H37gsmlW8Hzf0KtJ8F84z91Uq02vlpygNbDlrM65ixeFjMD7yvHrL4NqFIkX5amLyLiKC4tSU+aNMmV4V3LLxhafQCz+sDSj6DCA1DAeQseF8nvw/Bu1eg1Zh0T18VSrWg+utaMcFo8ERERERGnSb1qfw+9egQYVvAuAPd9DJUfvOlIHYDNsecZOH07e05dAqBR6SA+6FCZogWz/867IiK3kq3W3Mlzoh62r9qfdhV+fwkMw6nhGpcJZkCLMgC8OXMHO45fdGo8ERERERGHO7wKvmkAq4bZCzuVOsPz66BK15sWdi4np/Hu7J10+mY1e05dooCvB188VJVxj9dWYUdEcgUVd1zJZIK2w8DdCw4uhW2TnR7y+WalaF6uEClpNp79ZSMXElOcHlNEREREJNOSEuD3ATD2fjgXA/6h9gWTu4y2j4q/iYW74mg5dBljVx/GMKBT9XAWDmhCx2pFMN1ilI+ISE6i4o6rFYyEJq/YH88bCFfOOjWc2WxiaNcoihbw4dj5q7w4eQs2m3NHDImIiIiIZMq+BfB1Pdjwo/159V7w/Fr7gsk3cTohiT7jN/LkuA2cvJhE0QI+/PJEHYZ2jaKAr0cWJS4ikjVU3MkO6r8AhSrC1XMw/3Wnhwv0sfDNI9XxdDezdO8Zhi/e7/SYIiIiIiJ37MpZmP40THgQEo5B/uLQczY8MBy8Am94is1mMGFtLM2HLmPu9lO4mU082ySS+S82pmHpoKzNX0Qki6i4kx24Wey/oDDBtkkQs9jpISuGBfJBx8oAfLloP0v2nnZ6TBERERGR22IYsGM6fFXbvnSByQz1+sJza6Bkk5uedvZyMr3Hruf1Gdu5lJRG1SKB/Na3Ia/dVw5vD7csvAERkayl4k52UaQm1H7a/vj3lyAl0ekhu9QoQo86RTEMeHHSFo6ec35MEREREZFbSjgJk3rAtMcgMR6Cy8MT0dD6A/C4+eLH6w6d4/7hK1i+7wxeFjNvt63A9D4NqBAWkIXJi4i4hoo72UnztyAgHM4fhmUfZUnIt9tVoGpEPi5eTeW58RtJSrVmSVwRERERkesYBmz8Cb6qA3vngNkCTQfCM8vtH4TehM1m8NWSA3T//k/iEpKJDPZl1vMNebxhCdzMWjBZRPIGFXeyE09/aPO5/fHqkXBym/NDurvxdY/qFPD1YMfxBN6etcPpMUVERERErnPuEIx7AH57AZIvQngNe1Gn6WvgfvPFj89eTuaxsev5dP5erDaDTtXCmd23IWUL+2dh8iIirqfiTnZT9j6o0B4Mq/2Xm835I2nC83kzvFs1zCaYsuEYk9bFOj2miIiIiAg2K6z5yr4T1qHl4O4NrT6wT8MKqXDLU9cfPkeb4StZtu8Mnu5mPulchc+7VsXX0z2LkhcRyT5U3MmO7vsEPAPhxGZY+22WhGxYOoj/a1UWgLdn7WTbsQtZEldERERE8qjTu+HHVvbdYtOuQvFG0Gc11O8L5psvfmyzGXyzNIZu3/3JqYQk+zSsvg3oWisCk0nTsEQkb1JxJzvyLwwtB9kfL34fLmTNSJrnmkTSonwIKVYbz/2yifNXUrIkroiIiIjkIWkpsPRjGNUIjm8AzwBoNxx6/QYFSt7y1HNXUnjip/V8PG8PVptBh6gwZvdtSLnCWjRZRPI2FXeyq+q9oGg9SL0Cc162LzDnZGazic+7VqV4QR+OX7hK/8lbsNqcH1dERERE8ojjG+G7JrD0Q7ClQpn74Pm1UKMX/Meomw2Hz9Fm+AqW7LVPw/q4c2W+eChK07BERFBxJ/sym6Hdl+DmAfvnw87pWRI20NvCN4/UwMtiZvm+M3y5cF+WxBURERGRXCwlEea/AT+0gNO7wCcIuoyG7hMhIOyWp9psBqOWxfDQd39y8mISJYN8mfl8Ax6qVVTTsERE/qLiTnYWXBYa/Z/98R+vwtXzWRK2fGgAQzpVBmD44gMs2h2XJXFFREREJBc6tAK+qQ9rRoJhg8pd4fl1UKnzf47WOX8lhSfHbeCjP+zTsB6oGsbsfg0pH6ppWCIi/6TiTnbX8CUIKgNXzkD021kWtmO1IvSsVwyAlyZvIfZsYpbFFhERyc6KFy/O4MGDiY3V7pIit5R0EX7rDz+1hfOHICAcHp4Cnb8H34L/efrGI+dpM3wFi/ecxsPdzJBOlfmyWxR+moYlIpKBijvZnbunfYE5gE3j4PDKLAv9ZpsKVCuaj4SkNJ75ZSNXU5y/LbuIiEh29+KLLzJ9+nRKlixJy5YtmTRpEsnJya5OSyR72fsHfFUHNo61P6/5BPT5E8q0/s9TDcPgu+UxPPTtGk5cTKJEkC8z+zSge21NwxIRuRkVd3KCYvWgxmP2x7/1h9SkLAnr4W7m6x7VKejrwe6TCbw5cwdGFizsLCIikp29+OKLbNmyhXXr1lG+fHn69etHaGgoffv2ZdOmTa5OT8S1rsTDtCdgYje4dNK++1XvOdB2KHj991SqC4kpPDVuAx/O3UOazaBd1TB+69eQCmGahiUicisq7uQULd4FvxA4ewBWfJ5lYUMDvRnRvRpmE/y66RgT1mkIuoiICED16tUZPnw4J06c4J133uGHH36gVq1aREVFMXr0aH0gInmLYcC2qTCyFuyYBiYzNOgPz62G4g1v6xKbYs/TZvhKFu62T8P6oGMlhmsalojIbVFxJ6fwzgf3fWJ/vPILOLMny0LXLxXEK/eWA2DQ7F1sOXohy2KLiIhkV6mpqUyZMoUHHniA//u//6NmzZr88MMPdO7cmddff50ePXq4OkWRrHHxuH2kzvQn4eo5CKkETy6CloPB4v2fpxuGwffLD9J11BqOX7hK8YI+zOhTnx51imkalojIbVIZPCep0B7K3g975+I25yUIfj7LQj/TuCSbY88zf2ccfX7ZyG/9GlLQzzPL4ouIiGQXmzZtYsyYMUycOBGz2UzPnj354osvKFeuXPoxHTt2pFatWi7MUiQLGDbYMBoWvA0pl8DNAxq/Ag1fBDfLbV3iQmIKL0/dxsK/dmdtUyWUjzpVxt/r9s4XERE7FXdyEpMJ7v8UDi3HfHw9xU1LgLZZFNrEpw9WZX/cKg7GX6H/pC389Hht3Mz6NEVERPKWWrVq0bJlS7755hs6dOiAxZLxj9ASJUrQrVs3F2QnkjV8k07h9ksHiF1tbyhSCx4YCYXK3fK8f9oce56+EzZz/MJVPNzMvNWuAo/U0aLJIiJ3Q9OycprAItDcviV6hROT4fzhLAsd4GVh1KM18La4sfJAPEOj92ZZbBERkezi4MGDzJs3jwcffPCGhR0AX19fxowZk8WZiWQBmxXznyNptucNzLGrweID934Mj8+/7cKOYRj8uPIQXb+1T8MqVtCH6X3q82hdTcMSEblbKu7kRLWexBZWA4stCfef28GZfVkWukyIPx91rgzAV0tiWLDzVJbFFhERyQ5Onz7N2rVrM7SvXbuWDRs2uCAjkSyScBLGtcdt0bu4GanYSjSBPmug7rNgdrutS1xMTOXpnzfy3u+7SLUatKkcyu/9GlIpPNDJyYuI5G4q7uREZjesXcZyySsM06WTMOY+OLk1y8K3jwqnd/3iAPzflK0cir+SZbFFRERc7fnnn+fo0aMZ2o8fP87zz2fdengiWWr/QhjVEA6vwLD4srnoE1i7T4P8xW/7EluOXuD+4SuI3hWHh5uZ99pXZOTD1bS+joiIA6i4k1P5h7Ky9BsYhatAYjyMbQexGT9FdJbX7y9PjWL5uZScxnO/bCQxJS3LYouIiLjSrl27qF69eob2atWqsWvXLhdkJOJE1lSIfhvGd7a/5yxcmbQnFhFbsIl9PcjbYBgGo1ce4sFRqzl+4SpFC/jw63P1ebRecU3DEhFxEBV3crAUd3/SesyEovUg+SL83AFiFmdJbA93M1/3qE6Qnyd7Tl3ijRk7MAwjS2KLiIi4kqenJ3FxcRnaT548ibu79qqQXOT8EfsI8VVf2p/XegqeWAgFS932JS5eTeXZXzYy+K9pWPdVKszvLzSkchFNwxIRcSQVd3I6rwB4ZDpENofURJjwEOz+PUtChwR4MfLhariZTczYfJyf/zySJXFFRERcqVWrVgwcOJCLFy+mt124cIHXX3+dli1bujAzEQfa/Rt82wiOrQfPQOj6M7T5DCxet32JrUcv0HbECubvjMPiZmLQAxX5ukd1AjQNS0TE4VTcyQ08fKD7RCj/AFhTYEpP2Do5S0LXLVmQ1+6174zw3u+72BR7PkviioiIuMpnn33G0aNHKVasGM2aNaNZs2aUKFGCU6dO8fnnn7s6PZHMSU2Cuf+DyY9A0kUIrwnProAKD9z2JQzDYOyqQ3QZtZqj564SUcCbX5+rT6/6moYlIuIsKu7kFu6e0GUMVH0YDCvMeBrW/5AloZ9sVIL7Kxcm1WrQ55dNxF9OzpK4IiIirhAeHs62bdv45JNPqFChAjVq1ODLL79k+/btREREuDo9kbsXfwB+bAHrvrM/b9AfHp8H+Yvd9iUuXk3luV828e5v9mlY91YszO/9GlGlSD7n5CwiIgBoYnhu4uYO7b8CTz/7L+U5/wfJl6DhS04NazKZ+KRLVfaeukTMmSv0m7CZn5+ojbubaociIpI7+fr68vTTT7s6DRHH2TYFfn8JUi6DT0Ho+C2UvrNphtuPXeT5CZuIPZeIxc3E6/eXp7dG64iIZAkVd3Ibsxnu+wQ8A2DFZ7DwXUhKgOZv3/aOBnfDz9Odbx+twQMjV7Hm4Fk+W7CP1+4r57R4IiIirrZr1y5iY2NJSUm5rv2BB25/+oqIy6VcgbmvwJZf7M+LN4JO30NA6G1fwjAMxq05wgdzdpNitVEkvzdfPVydqhH5nJOziIhkcFfFnaNHj2IymShSpAgA69atY8KECVSoUEGfYmUHJhM0fws8/WHhO7ByqH0Ez32f2Is/TlKqkD+fdKlC3wmbGbUshqiIfNxbqbDT4omIiLjCwYMH6dixI9u3b8dkMqXvFvn36ASr1erK9ERuX9wumNob4veCyQxNXoXG/wOz221fIiEpldd+3cbc7acAaFUhhE+7VCXQR4smi4hkpbv6S//hhx9myZIlAJw6dYqWLVuybt063njjDQYPHuzQBCUTGr4IbT4HTLD+e5jVB6xpTg3ZtkoYTzQsAcDLU7dy8Mxlp8YTERHJav3796dEiRKcPn0aHx8fdu7cyfLly6lZsyZLly51dXoi/80wYMMY+L6ZvbDjVxh6zoamr91RYWfniQTaDl/J3O2nsLiZeLttBb59tIYKOyIiLnBXxZ0dO3ZQu3ZtAKZMmUKlSpVYvXo148ePZ+zYsY7MTzKr1pPQ6TswucHWiTCtN6Q5d8Hj1+4rR+3iBbicnMbTP2/kdEKSU+OJiIhkpTVr1jB48GCCgoIwm82YzWYaNmzIkCFDeOGFF1ydnsitJV2EaY/B7y9CWhKUagnPrYISjW77EoZhsOKUiQe/W0vsuUTC83kz9dn6PN6whNbXERFxkbsq7qSmpuLp6QnAwoUL0+eWlytXjpMnTzouO3GMKl3hoZ/BzQN2/wYTu9nnVzuJxc3MyB7VKOTvyYHTl+nw1Sp2nUhwWjwREZGsZLVa8ff3ByAoKIgTJ04AUKxYMfbu3evK1ERu7fgm+LYx7JwBZndoORgengK+Qbd9iaspVl6cso1ph9xItRq0rBDC3BcaEaX1dUREXOquijsVK1Zk1KhRrFixgujoaO69914ATpw4QcGCBR2aoDhIuTb2X94WH4hZDD93sn9y4ySF/L2Y8kw9Sgb7cuJiEl1GrWbhrjinxRMREckqlSpVYuvWrQDUqVOHTz75hFWrVjF48GBKlizp4uxEbsAwYM1X8GMrOH8YAovCY/PsW53fwXqMqVYbz43fyNwdcZhNBq/fV5bvNA1LRCRbuKvizscff8y3335L06ZN6d69O1WrVgVg9uzZ6dO1JBuKbAaPzgTPQDj6J/zUDq7EOy1c8SBfZjzXgAalCpKYYuWpnzfww4qD6QtPioiI5ERvvvkmNpsNgMGDB3Po0CEaNWrE3LlzGT58uIuzE/mXxHMwsTvMfx1sqVC+HTy7HCJq3dFlbDaDV6dtY+neM3hZzDxfwcpj9YtpGpaISDZxV7tlNW3alPj4eBISEsifP396+9NPP42Pj4/DkhMnKFoHev8OP3eEk1thzP3QcyYEhDklXKCPhbGP1ebtWTuZuC6W9+fsJubMFQa3r4jFzXk7d4mIiDhL69at0x+XKlWKPXv2cO7cOfLnz68/dCV7ObIGfn0CEo7bp+e3/tC+HuNd/Jx+PG8P0zcfx81sYvhDVbkas94JCYuIyN26q7+ur169SnJycnph58iRIwwbNoy9e/dSqFAhhyYoThBaBR77AwLC7TskjL4Xzh1yWjiLm5kPO1bizTblMZlg4rpYHhuznotXU50WU0RExBlSU1Nxd3dnx44d17UXKFBAhR3JPmxWWP4pjG1jL+wUiIQnF0Ltp+6qsPP98oN8u/wgAB93rkKzssGOzlhERDLproo77du3Z9y4cQBcuHCBOnXq8Pnnn9OhQwe++eYbhyYoThJcxl7gyV8CLhyxF3hO73FaOJPJxJONSvLdozXx8XBj5YF4On29iiNnnbews4iIiKNZLBaKFi2K1Wp12DW/+uorihcvjpeXF3Xq1GHdunW3dd6kSZMwmUx06NDBYblILnApDn7pBIvfB8MKlbvCM8sgtOpdXW76pmN8MHc3YN8RtUuNIo7MVkREHOSuijubNm2iUSP7donTpk0jJCSEI0eOMG7cOM01z0nyF4PH50GhCnD5FIy5D05sdmrIlhVCmPpsPUIDvYg5c4UOX61i3aFzTo0pIiLiSG+88Qavv/46585l/vfX5MmTGTBgAO+88w6bNm2iatWqtG7dmtOnT9/yvMOHD/Pyyy+nvx8TAeybZoxqAAeX2jfRaP8VdPoOPP3v6nJL9p7mlWnbAHiyYQmeaawFw0VEsqu7Ku4kJiambwG6YMECOnXqhNlspm7duhw5csShCYqT+ReG3nMgrDpcPQdj28GR1U4NWTEskFnPN6BKkUDOJ6byyA9r+XXjMafGFBERcZSRI0eyfPlywsLCKFu2LNWrV7/u604MHTqUp556iscee4wKFSowatQofHx8GD169E3PsVqt9OjRg0GDBml3LrGzpsGiwfbdUK+csX9w99QSqPbIXU3DAtgce54+v2wizWbQISqM1+8vr6mHIiLZ2F0tqFyqVClmzpxJx44dmT9/Pi+99BIAp0+fJiAgwKEJShbwKQA9Z9l3Ujiy0v7G4KFfoHQLp4UsFODF5KfrMWDKFv7YcYr/m7qVQ/FXGNCyDGaz3jiIiEj25ahpUCkpKWzcuJGBAwemt5nNZlq0aMGaNWtuet7gwYMpVKgQTzzxBCtWrPjPOMnJySQnJ6c/T0hIAOzrB6Wm5q317/6+31x13wnHcZvxNOZjawGwVuuJreUHYPGGu7zPmDNXeHzseq6mWmlUqiAftK+A1ZrG37MRc2U/ZjH1oWOoHzNPfegYzurHO7neXRV33n77bR5++GFeeukl7rnnHurVqwfYR/FUq1btbi4pruYVAI9Mgyk9Yf8CmNgNuvwIFdo7LaS3hxtfPVydzxbs5eulMYxccoBD8Vf47MGqeHu4OS2uiIhIZrzzzjsOuU58fDxWq5WQkJDr2kNCQtiz58br4K1cuZIff/yRLVu23HacIUOGMGjQoAztCxYsyLO7nEZHR7s6BYcofHET1Y58j9l6hVSzF1uKPs4J6kL0kru+5oVk+GKHGxdSTBTzM2hXII6FC+bd8Njc0o+upD50DPVj5qkPHcPR/ZiYmHjbx95VcadLly40bNiQkydPUrXqtcXZmjdvTseOHe/mkpIdWLzhofEw/SnYNROm9oYHRkK1Hk4LaTabeOXecpQM9mPg9G3M2X6SY+cT+b5XTQr5ezktroiISE5z6dIlHn30Ub7//nuCgoJu+7yBAwcyYMCA9OcJCQlERETQqlWrPDfiOjU1lejoaFq2bInFYnF1OncvLRnz4sG4HfwWAFvhqtDpB6LylyAqE5e9eDWV7j+s40LKFUoG+TDxydoU8PXIcFyu6UcXUh86hvox89SHjuGsfvx7tO3tuKviDkDhwoUpXLgwx47Z10opUqQItWvXvtvLSXbh7gFdRsNv/rD5Z5jVB1IuQ51nnBq2S40iROT35plfNrL12EU6jFzFD71qUSEsb73pFBGR7M9sNt9y7ZHb3UkrKCgINzc34uLirmuPi4ujcOHCGY6PiYnh8OHDtGvXLr3NZrMB4O7uzt69e4mMjMxwnqenJ56enhnaLRZLnn0jn6Pv/dxBmPoYnNxif163D+YW72J2z/g9vhNXU6w8O34L+09fISTAk3FP1CEk361HduXofswm1IeOoX7MPPWhYzi6H+/kWne1oLLNZmPw4MEEBgZSrFgxihUrRr58+XjvvffS32RIDmZ2gwdGQN0+9ud/vALLPwXDcGrYOiULMrNPA0oG+XLiYhIPjlrNot1x/32iiIhIFpoxYwbTp09P/5o8eTKvvfYaoaGhfPfdd7d9HQ8PD2rUqMGiRYvS22w2G4sWLUqf8v5P5cqVY/v27WzZsiX964EHHqBZs2Zs2bKFiIgIh9yfZGPbp8GoxvbCjnd+6D4J7h0CmSzspFlt9Ju4iQ1HzhPg5c5Pj9emSP68OWVPRCSnuquRO2+88QY//vgjH330EQ0aNADsc8DfffddkpKS+OCDDxyapLiAyQStPwTPAFj2ESx+H5ISoOXgu9514XYUD/JlRp8GPDd+I6tjzvLUuA280aYCjzcorh0aREQkW2jfPuN6dF26dKFixYpMnjyZJ5544ravNWDAAHr16kXNmjWpXbs2w4YN48qVKzz22GMA9OzZk/DwcIYMGYKXlxeVKlW67vx8+fIBZGiXXCYlEea9Bpt+sj+PqGtfGzGwSKYvbRgGr8/YzsLdp/F0N/NDr1qUK6yR0yIiOc1dFXd++uknfvjhBx544IH0tipVqhAeHk6fPn1U3MktTCZoNhA8/WHBG7B6uH2K1v2fg/muBn3dlkAfCz89Xpu3Z+1g4rqjvPf7LmLOXGbQAxWxuDkvroiISGbUrVuXp59++o7Oeeihhzhz5gxvv/02p06dIioqinnz5qUvshwbG4vZib9zJQc4vce+DuKZ3YAJGv0fNB0Ibne9usJ1Pp2/lykbjmE2wYju1ahdooBDrisiIlnrrn4rnDt3jnLlymVoL1euHOfOnct0UpLN1O8Lnn7w24uwYTQkX4YOX4Ob8+ZkWtzMfNixMiWD/Pjwj91MWBvL0XOJjHy4OoHemgsqIiLZy9WrVxk+fDjh4eF3fG7fvn3p27fvDV9bunTpLc8dO3bsHceTHMIwYPMvMPd/kHYVfAtBp+8gspnDQoxeeYivl8YA8GHHyrSqmHGtJxERyRnuqrhTtWpVRo4cyfDhw69rHzlyJFWqVHFIYpLN1OgNHn4w4xnYPgVSrtgXXrY4b0crk8nEU41LUjzIl/6TNrNifzydv1nN6F61KFpQ88BFRMQ18ufPf91UYcMwuHTpEj4+Pvzyyy8uzExyjeRL8PsA+3sugJJNoeN34B/isBCzt55g8O+7APhf67J0q13UYdcWEZGsd1fFnU8++YQ2bdqwcOHC9AX/1qxZw9GjR5k7d65DE5RspHIXe4FnSk/YOwcmdIVuE+yjepyoZYUQpjxTjyd/2sCB05dp/9VKvutZk1rFNWxYRESy3hdffHFdccdsNhMcHEydOnXInz+/CzOTXOHkVvtuWOdiwOQGzV6HhgMcOiV+xf4z/N+ULQD0rl+cPk0z7rImIiI5y10Vd5o0acK+ffv46quv2LNnDwCdOnXi6aef5v3336dRo0YOTVKykbL3wiPTYGJ3OLQMfu4IPaaCdz6nhq0UHsisvg148qcNbD9+kR7fr+WjzpXpVD3zCwmKiIjcid69e7s6Bcmtzh2E0fdCaiIEhEPnH6FYxp3TMmPr0Qs88/NGUq0GbaqE8nbbCtq0QkQkF7jrjwDCwsL44IMP+PXXX/n11195//33OX/+PD/++KMj85PsqERj6DkLvPLBsXUwti1cPuP0sCEBXkx+pi73VixMitXGgClb+Wz+Xmw2527RLiIi8k9jxoxh6tSpGdqnTp3KTz/95IKMJNeY/6a9sFOkNjy70uGFnYNnLvPY2PUkplhpUKogQ7tWxWxWYUdEJDfQ9gtyd4rUhN5z7Iv7xW2HMffCxWNOD+vj4c7XParz3F/Dh0cuOUC/SZtJSrU6PbaIiAjAkCFDCAoKytBeqFAhPvzwQxdkJLnCgYX2ae8mN3hgBPg4dvr56YQkeo5ex7krKVQOD+TbR2vi6e7m0BgiIuI6Ku7I3StcCR6fB4ERcPaAfRjx2RinhzWbTbx6bzk+6VIFi5uJOdtO8tB3f3L6UpLTY4uIiMTGxlKiRIkM7cWKFSM2NtYFGUmOl5YCf7xmf1znGSiUcVfazLh4NZWeo9dx7PxVihf0YcxjtfDzdMxW6iIikj2ouCOZUzASHvsDCkTCxaMw5j6I25klobvWjODnJ+qQz8fC1qMX6DByFbtPJmRJbBERybsKFSrEtm3bMrRv3bqVggULuiAjyfHWfQdn94NPEDR51aGXTkq18tS4Dew5dYlgf0/GPV6HID9Ph8YQERHXu6OSfadOnW75+oULFzKTi+RU+SLsI3h+7ghxO2DM/fDIdChSw+mh65YsyIw+DXhi7HoOxl+hyzerGfFwNe4p57itQkVERP6pe/fuvPDCC/j7+9O4cWMAli1bRv/+/enWrZuLs5Mc5/JpWPax/XGLdxy6SYXVZtB/0mbWHTqHv6c7Yx+rRdGCPg67voiIZB93NHInMDDwll/FihWjZ8+ezspVsjO/QtD7dyhSC5IuwLgH4NCKLAldIsiX6X3qU69kQa6kWHnypw2MXnkIw9BCyyIi4njvvfcederUoXnz5nh7e+Pt7U2rVq245557tOaO3LlFgyA5AUKjIOoRh13WMAzenLmD+Tvj8HAz813PmlQMC3TY9UVEJHu5o5E7Y8aMcVYekht454dHZ8Kk7nBoOYzvAl3HQZnWTg+dz8eDcU/U5q2ZO5i0/iiDf9/FwfjLvNOuIhY3zT4UERHH8fDwYPLkybz//vts2bIFb29vKleuTLFixVydmuQ0xzfC5l/sj+/7BMyOe8/yxcL9TFwXi8kEX3aLol6kpgyKiORmWklNHMvTDx6eClN7w74/YMJDUO0RaP4O+AU7NbTFzcyQTpWJDPbjwz9288ufsRw5m8jIh6sT6G1xamwREcl7SpcuTenSpV2dhuRUNhv88df6OlW6QdE6Drv0z2sOM3zRfgDea1+J+yqHOuzaIiKSPWlIgziexQse+hmq9wIM2PwzjKgOa74Ca6pTQ5tMJp5qXJJvH6mBt8WNFfvj6fzNamLPJjo1roiI5B2dO3fm448/ztD+ySef8OCDD7ogI8mRtk2GY+vB4gst3nXYZedsO8nbs+2bW7zYojSP1NWIMhGRvEDFHXEONws8MBweX2CfQ56cAPNfh2/qw4GFTg/fqmJhpj5bj8IBXhw4fZkOX69iw+FzTo8rIiK53/Lly7n//vsztN93330sX77cBRlJjpOUAAvfsT9u8j8IcMzImtUH4nlp8hYMA3rUKUr/5hpZJiKSV2Sb4s5HH32EyWTixRdfdHUq4khF68BTi6HdcPv2nvH74JfOMKEbnI1xauhK4YHMfL4BlcIDOHclhYe/X8uMzcecGlNERHK/y5cv4+HhkaHdYrGQkJDggowkx1n+KVyOgwIloW4fh1xyx/GLPP3zRlKsNu6rVJjB7SthMpkccm0REcn+skVxZ/369Xz77bdUqVLF1amIM5jdoEYv6LfR/gbG7G5fj+frurDwXUi+7LTQhQO9mPJMPVpXDCHFauOlyVv5fMFebDbtpCUiInencuXKTJ48OUP7pEmTqFChggsykhwl/gD8+Y398b0fgbtnpi955OwVeo9Zz+XkNOqWLMAXD0XhZlZhR0QkL3H5gsqXL1+mR48efP/997z//vu3PDY5OZnk5OT0539/OpaamkpqqnPXcslu/r7fHHXf7r7QfDBUfQS36DcwH1wCK7/A2DIR6z1vY1R6EJzwCZPFBMO7VuHzhfv5bsVhRiw+QMzpS3zcqRJu2IAc1o/ZTI78WcyG1I+Zpz50DGf1Y276vrz11lt06tSJmJgY7rnnHgAWLVrEhAkTmDZtmouzk2xv/kCwpUKplg7ZUfTMpWQe/XEd8ZeTKR8awHc9a+JlcXNAoiIikpO4vLjz/PPP06ZNG1q0aPGfxZ0hQ4YwaNCgDO0LFizAx8fHWSlma9HR0a5O4e4E9KZwiSpUOj4R38uncJ/dh3OLhrK9yCNc8CnplJAVge6RJiYfNDN3Rxw7D5/iybJWAjxycD9mI+pDx1A/Zp760DEc3Y+JiblnYft27doxc+ZMPvzwQ6ZNm4a3tzdVq1Zl8eLFFChQwNXpSXa2bz7sXwBmC9w7JNOXu5SUSu8x64g9l0hEAW9+eqwWAV7aIVREJC9yaXFn0qRJbNq0ifXr19/W8QMHDmTAgAHpzxMSEoiIiKBVq1YEBAQ4K81sKTU1lejoaFq2bInFklN/ibeBtP9hXTsK86ovKHDlAI33DsKo+jDWZm+Cr+O3Tr8faHPoHH0nbuXI5VS+PuDLo8Wu0LtDTu5H18odP4uup37MPPWhYzirH3PbWjRt2rShTZs2gP3eJk6cyMsvv8zGjRuxWq0uzk6ypbRkmDfQ/rjucxCUucWOk9OsPPPzRnaeSKCgrwfjHq9DoQAvByQqIiI5kcuKO0ePHqV///5ER0fj5XV7v4g8PT3x9Mw4L9liseTZN/I5/t4tFmj6P6jeA6LfwbR9Cqat4zHv+Q2avga1n7bvvOVADcuEMOP5Bjw+dj2H4q/w+XY3UoOP0uee0ljcssUyVDlSjv9ZzCbUj5mnPnQMR/djbvyeLF++nB9//JFff/2VsLAwOnXqxFdffeXqtCS7+vMbOBcDvoWg8f8ydSmrzeClyVtYHXMWXw83xj5WmxJBvg5KVEREciKX/SW7ceNGTp8+TfXq1XF3d8fd3Z1ly5YxfPhw3N3d9alXXhMQBp2/h8fnQ2jVf22dvsjh4UoE+TKjT32algnCapj4YtEBHhi5iu3HLjo8loiI5B6nTp3io48+onTp0jz44IMEBASQnJzMzJkz+eijj6hVq5arU5TsKOGkfYcsgJaDwOvuR5wbhsGg33Yyd/spLG4mvn20JpWLBDooURERyalcVtxp3rw527dvZ8uWLelfNWvWpEePHmzZsgU3Ny0ElycVrQtPLfnX1umdYGJ3OHfQoaHy+Xjw3SPVeLSUlfw+FnafTKDD16v46I89JKWquCgiItdr164dZcuWZdu2bQwbNowTJ04wYsQIV6clOcHCdyHlMoTXhCrdMnWpkYsPMG7NEUwmGNo1ioalgxyTo4iI5GguK+74+/tTqVKl6758fX0pWLAglSpVclVakh3caOv0vXPhqzqwcJBDt043mUzUDDb4o1992lUNw2ozGLUshvu+XMHag2cdFkdERHK+P/74gyeeeIJBgwbRpk0bfRAlt+foOtg2yf74/k/AfPdvvyesjeXz6H0AvNO2Au2qhjkiQxERyQW0wIhkX9757DtJPLcaIu8BawqsHAoja8LWyWAYDgtV0M+TEd2r8X3PmoQEeHIo/goPffcnb87czqWk3LN9r4iI3L2VK1dy6dIlatSoQZ06dRg5ciTx8fGuTkuyM5sN5v61vk7UIxBe464vNW/HKd6cuR2Avs1K0btBCUdkKCIiuUS2Ku4sXbqUYcOGuToNyW6Cy8Ij06HbBMhfHC6dhBlPw+jWcGKzQ0O1rBBC9IAmdK8dAcAvf8bS+ovlLNlz2qFxREQk56lbty7ff/89J0+e5JlnnmHSpEmEhYVhs9mIjo7m0qVLrk5Rspstv8DJLeAZAC3euevLrD14lhcmbcZmQLdaEfxfqzKOy1FERHKFbFXcEbkpkwnKtYE+a+Get8DiA0fXwnfNYHY/uHzGYaECvCwM6VSFCU/WoWgBH05cTOKxset5afIWzl1JcVgcERHJmXx9fXn88cdZuXIl27dv5//+7//46KOPKFSoEA888ICr05Ps4uoF+3RygCavgl+hu7rM7pMJPDluAylpNlpWCOH9DpUwmUyOy1NERHIFFXckZ7F4QeOX7evxVO4KGLBpHIyoAWu+AqvjplDVLxXE/Bcb81SjEphNMGPzcVoOXcZvW09gOHBKmIiI5Fxly5blk08+4dixY0ycONHV6Uh2suwTSIyHoDJQ++m7usTRc4n0HL2OS0lp1CqenxHdq+HuprfvIiKSkX47SM6UYev0i07ZOt3bw4032lRgep8GlA3x5+yVFPpN3MxT4zYSl5DksDgiIpKzubm50aFDB2bPnu3qVCQ7OL0H1n1rf3zvEHD3uONLnL2cTM/R6zhzKZmyIf780LMWXhYt4i0iIjem4o7kbFm0dXpURD5+69eQF1uUxuJmYuHuOFoMXcakdbEaxSMiIiLXGAbMew1saVD2fijV4o4vcTk5jcfGrudQ/BXC83nz0+O1CfSxOCFZERHJLVTckZzv31unm9ycsnW6h7uZF1uU4fd+jagakY9LSWm8Nn07D3+/liNnrzgkhoiIiORwe+fCwSXg5gGtP7jj01PSbDz3y0a2HbtIfh8L456oTeFALyckKiIiuYmKO5J7/HPr9JLNrt86fdsUh22dXrawP9Ofq8+bbcrjZTGz5uBZWg9bzg8rDmK1aRSPiIhInpWaBPMG2h/X6wsFSt7R6YZh8L9pW1mxPx5vixtjHqtNZLCfExIVEZHcRsUdyX0KlYNHZ1y/dfr0pxy6dbqb2cSTjUoy/8XG1I8sSFKqjffn7KbTN6vZe0pb4YqIiORJa0bAhSPgHwqN/u+OT5+47iiztpzA3Wxi1KM1iIrI5/gcRUQkV1JxR3KnLNo6vVhBX8Y/WYePOlXG39OdrUcv0HbECoYt3EdKms0hMURERCQHuHgcVgy1P275Hnje2Yib2LOJvD9nFwCv3VeOJmWCHZ2hiIjkYiruSO52y63Tv3bI1ukmk4lutYsSPaAJLcqHkGo1GLZwP+1GrGTL0QuZvr6IiIjkANFvQ2oiRNSFyl3u6FSrzeDlqVtJTLFSu0QBHm9QwklJiohIbqXijuQNN9w6fSDuPzQh5OIWh6zHUzjQi+971mDkw9Uo6OvB3rhLdPp6Fe//vourKdbM34OIiIhkT0dWw45pgAnu/8Q+gvgOjF55iHWHz+Hr4cbnD1bFbL6z80VERFTckbwlfev0L8GnIKb4fdQ9OBS3Ma1g77xMF3lMJhNtq4SxcEATOlYLx2bADysP0XrYclbHxDvoJkRERCTbsFlh7iv2xzV62T9EugP74i7x6YK9ALzZtgIRBXwcnaGIiOQBKu5I3mN2gxq9od8mrHWfJ83kgfnkZpj4EHzXBPbMyXSRJ7+vB188FMWY3rUIDfQi9lwiD3+/loHTt5GQlPmpYCIiIpJNbPoJ4raDV6B9nb87kGq1MWDKFlLSbDQtG0y3WhFOSlJERHI7FXck7/LOh635IKIrDsVarx9YfOHkVpj0MIxqBLtmgy1ziyI3K1eIBS815tG6xQD7Lhgthy4jelecI+5AREREXCnxHCx6z/646evgG3RHp49cfIAdxxMI9LbwcecqmO5wOpeIiMjfVNyRPC/FEoDtnnfgxe3QcAB4+Nk/gZvyKPx/e/cdHlW19XH8OzPpvUAahJIQSGihBJCOVFFBsKGiYm+gIHJFVGxXRVAR24u9XUUpCiIIGHrv0oRQA4SShJpK6sz7x0gkIBCSSSZDfp/nOU9mzpxzZp0dSHbW7L32px3hrxllSvJ4uznz336NmfzINdSt5klKei4Pf7eeIZM2cjwz13Y3IiIiIhVr8Rg4cxKqx0CrB6/o1C2HTvPRoj0A/LdfY4J93MojQhERqSKU3BE5yzMQuv+d5Ok4Aly8IWUbTB0EE9vBtp+t8+pLqU1EIHOGduSxzpGYjAZmbTlKj/FLmPHnYSw2KOgsIiIiFSjlL1j3pfVx77fA5FziU3PyCxk+ZTOFZgs3NA2lb2xYOQUpIiJVhZI7IufzCIBuo+HprdB5JLj6wrEdMO0B+L+2sGVqqZM8bs4mnusdzYwn2hMT6sOp7HyGTd7EA9+s48jpMza+ERERESkXFgvMGQmWQojpCxFdruj0d+btZE9qJtW9XXn9psblE6OIiFQpSu6IXIy7P1z7PAzbYp1H7+YLx3fCLw/Bx21g82QoLCjVpZvU9GXmkPb8p1cDXExGFu08Rs/3lvK/1QcwmzWKR0REpFLb/ivsXwZObtDz9Ss6dfW+E3y5IhGAsbc0wd/TpTwiFBGRKkbJHZHLcfeDLiOt07W6vmhN+pzYDdMfgY9bw6ZJpUryOJuMDL62Hr8P7UCLWn5k5hYwesY27vh8NfuOZdr+PkRERKTs8rLhjxetj9sPBf/aJT41M7eAEVM3Y7HAgLhwukYHl1OQIiJS1Si5I1JSbr7Q6T/WJE+3l8E9AE7uhRmPw0ctYeP/oPDKlzmvF+TN1Mfa8Uqfhni4mFibeJLe7y/jkyV7KSgs22pdIiIiYmMrP4C0JPCpCe2HXdGpb8zezqFTZ6jh586LN8aUT3wiIlIlKbkjcqVcvaHjcGuSp/ur4FENTu2HmUPgwxaw4RsoyLuiS5qMBu5rX5d5wzrRMaoauQVm3pqTQL//W8H2I+nlchsiIiJyhU4fhOXvWR/3/C+4eJT41EUJqfy4NgmAd26Lxdut5AWYRURELkfJHZHScvWCDsOsNXl6vg6e1a2dvt+GWpM8676Egitb6jw8wIPvHmjN27c2xcfNiW2H0+n70XJen7Wd09lXljASERERG/vjRSjIgdodoFH/Ep92OjuPkT9vAeCB9nVpGxlYXhGKiEgVpeSOSFm5eEK7J2HoFug1BryCrcO1Zw+HD5rD2s8hP6fElzMYDNwWF878ZzrTu3EIBWYLXyxPpPPbi/l86T5yC0q/HLuIiIiUUuJSayFlgxF6jwWDocSnjv71L1Izcoms7smz1zUoxyBFRKSqUnJHxFZcPKDtEzB0M/QeB96hkH4Yfh9hTfKs+fSKkjxB3m5MvLsl39zfigbB3qSdyeeN33fQ7d0l/LrpsFbVEhERqSiFBdalzwHiHoSQki9fPmvLEX7bfAST0cD425vh5mwqpyBFRKQqU3JHxNac3aHNo/DUJrj+HfCpARlHYM6z8H4srPo/yD9T4st1aRDE70M7Mu6WpgT7uHLo1BmG/rSJfv+3gtX7TpTffYiIiIjV+q8gdbt1xcxrny/xaanpObw4YxsAg7tEEhvuV04BiohIVafkjkh5cXaD1g/DU3/CDePBNxwyk2HeKJjQFFZ+BHlZJbqUyWjg9lbhLBrRhWd61MfTxcSWQ2nc8dlqHvp2HXtSM8r5ZkRERKqorBOw6HXr464vgkdAiU6zWCw898tWTmfn0yjMhyFdo8oxSBERqeqU3BEpb06u0OpBeHIj9Hkf/GpBVir88YI1ybPifcjNLNGlPFyceLJbFIv/cy13X1MLk9HA/B2p9JqwjOenbyU1o+TTvkRERKQEFr0OOWkQ3ARa3l/i06asT2JhQiouJiPjb2+Gi5O63SIiUn70W0akoji5QMv7rEmevh+Bfx3IPg7xL8H7TWHZeMgt2Qic6t6uvN6vCfOGdaJHw2AKzRYmrTlIl7cX8/783WTnFZTrrYiIiFQJRzfD+q+tj3uPBWPJ6uUknczmtd+2AzC8Z30ahHiXV4QiIiKAkjsiFc/kDC3ugSHr4ab/g4AIyD4BC16FCU1g6duQk16iS9UL8uLze+OY8mhbYsP9yM4r5L35u+j89mJ+XHuQgkJzOd+MiIjIVcpi+buIsgUa3Qx12pfoNLPZwn+mbSYrr5C42v483DGifOMUERFByR0R+zE5Q/OBMHgd9P8UAuvBmVOw8HWY0BgWj4Uzp0t0qdZ1A5jxRDs+uqs5tQI8OJaRy6hftnL9B8tYmJCCxaKVtURERK7Itp/h4Cpwcoee/y3xad+s3M/qfSdxdzbxzm2xmIwlXzJdRESktJTcEbE3kxPE3gGD18LNX0C1+ta5/YvftNbkWTQGso5f9jIGg4Ebm4YRP7wTo29siJ+HM7tSMnngm/Xc9fkath1Oq4CbERERuQrkZcEfo62POz4DvjVLdNqe1EzGzk0A4PkbYqhTzbO8IhQRESlGyR2RysJogqa3wROr4davoHoM5KbBkrfg3WiYMgj2LgTzpadauTqZeLBDXZaMuJZHO0Xg4mRk1b4T3Pjhcob99CeHTmVX0A2JiIg4qGXjIeOIdRGEdkNKdEpBoZlnpm4mt8BMx6hq3N2mVjkHKSIi8g8ld0QqG6MJGt8Cj6+E276FsOZgzoftM+B//eGDZta6POlHLnkZXw9nRl0fw8JnOtOvWRgAMzYdoeu7Sxjz+w7SzuSX/72IiIg4mpOJsPJD6+Neb4Kze4lOm7h4L5uTTuPt5sS4W5tiMGg6loiIVBwld0QqK6MRGvWDRxbDo8ug1cPg6gunD1jr8rzXCCbdATvnQOHFV8eq6e/BhDua89uQDrSNCCSvwMynS/fR+e1FfLk8kbwCFV0WEREp8seLUJgLEV0g+sYSnbLtcBrvL9gNwGs3NSLUt2QJIREREVtRckfEEYQ2hRvegWcSoN8nUKsdWMywaw78eIe1APOC/8Kp/Re9RJOavkx6uA1f3RdHVJAXp7Pz+e+s7XQfv4RZW46o6LKIiMieBZAwCwwmuG4slGD0TW5BIc9M2UyB2cJ1jULo16xGBQQqIiJSnJI7Io7ExQOa3QkPzLGustV2CHgEQsZRWPYOvB8L3/WDbb9AQe4FpxsMBrpGBzNnaEfG3NyE6t6uHDyZzZBJf9L//1aybv/Jir8nERGRyqAwH+aOsj5u/QgERZfotPHxu9iZkkE1Lxfe6N9Y07FERMQulNwRcVTV60OvN2B4Atz2DURca92/bxFMux/Gx8C8F+DYrgtOdTIZubN1LRaP6MKw7lF4uJjYlHSa2z5ZxSPfrWfvscyKvRcRERF7W/s5HN9p/dCky3MlOmX9/pN8tnQfAG/2b0Kgl2t5RigiInJRSu6IODonF2jUH+6dAUM3Q6f/gHcoZJ+AVR/Bx63gq+tg04+QV3ylLE9XJ4Z1r8/i/3Thrja1MBrgj+0p9HxvKS/O2MqxjAtH/4iIiFx1Mo/B4jHWx91eBne/y56SlVvAM1M3Y7HALS1q0rNRSPnGKCIicglK7ohcTfzrQNcXYdg2uPMnaHC9tW7AwVUw4zHrkuqzn4GjW4qdFuTtxpv9m/DH053oHhNEodnC96sP0uXtRXy4YDdn8grtcz8iIiIVYcGrkJsOobHQ/O4SnTJmzg4OnMgmzNeNl/s2LOcARURELk3JHZGrkckJGvSGO3+Ep/+yJnz8akNuGqz7Aj7tCJ92hvVfQU560Wn1grz5YlArfnz4GprW9CUrr5B343fR5Z1FTFmXRKFZRZdFROQqc3gj/Pm99XHvcWA0XfaUpbuO8f3qgwC8fVssPm7O5RmhiIjIZSm5I3K18wm1TtV6ahPcM8M6hcvoDEc3wayn4d0GMGMwJK2Fv1fMahsZyIwn2vP+Hc2o6e9OSnouz/68hevfX8ainalaWUtERK4OZjPMGQlYoOkAqHXNZU9Jy87n2WnWEbCD2tamfb1q5RykiIjI5TnZOwARqSBGI0Rea92yTsDmH2Hjt3B8F2z63rpVj4EW90LsHRg9AripWQ2uaxzCdysP8OHC3exMyeD+r9fRvl4go3rH0LiGr73vSkREpPS2ToFDa8HZE7q/WqJTXvntL5LTc6hbzZPneseUc4AiIiIlo5E7IlWRZyC0GwKD18ID8yD2LnByh2M7YN4o62ieaQ/CviW4Gg083CmCpc9ey0Md6uJiMrJizwn6fLSc4ZM3ceT0GXvfjYiIyJXLzYD4l6yPO42wjnS9jLnbjjL9z8MYDfDu7bG4u1x+CpeIiEhFUHJHpCozGKxD0PtPhGcS4IZ3IaQpFObBtmnwXV/4sAUsG49f4SlevLEhC57pTJ/YMCwW+OXPw/R4fwUzDxg5lZ1n77sREREpuaXvQGYK+NeFtoMve/ixjFyen74NgMc6R9Kiln95RygiIlJiSu6IiJW7H7R6CB5bBo8shpb3g4s3nEq0riIyPgZ+Gkj48eV8OKApvw5uT+u6AeQVmFlwxEiXd5fx+qztpKTn2PtORERELu3kXlj1sfXxdW+Bk+slD7dYLDw/fSsns/KIDvFmaPeoCghSRESk5JTcEZELhTWHPhNgxE646WOo2RoshZAwCybdBhOaELvn/5h8exif3NWMmp4WsvMK+WJ5Ih3HLmLUL1s5cCLL3nchIiLyr0zxL4I5H+p1h/q9Lnv8zxsPE789BWeTgfG3N8PVSdOxRESkclFBZRG5OBdPaH63dUvdARu/sxZiTj8MS8ZiWDKOnhFdCQiPIav5fXy88hjrD5zix7UHmbzuIH1iw3iiSz0ahHjb+05EREQACErbjHFfPBidrKN2DIZLHn/49BlenfkXAMO616dhmE9FhCkiInJFNHJHREomKAauGwPDE+CWL6FuJ8CCcd8CWu//iC4z2zHN4y2Wtt/KXXUyMVss/LrpCL0mLOXh79azKem0ve9ARESqusI8mhz+3vr4mseh2qWnV5nNFp6dtpmM3AKa1/Lj0U4RFRCkiIjIldPIHRG5Ms5u0ORW63ZiL4UbviVn/SQ8845B4hJqJS7hTeDVwFDWmJrzw4n6LN/emPjtKbSvF8jgLvVoGxmI4TKflIqIiNiacc0neOWmYPEMwtDp2cse/7/VB1ix5wRuzkbevS0WJ5M+FxURkcpJyR0RKb3ASMzXjmZ+dguubxuNc+Ji2DMf9i/DOesoHThKB+ffKXQ2scEcxeLEpryxtxkuNZryRNf6dIsOwmhUkkdERMqZxQIrP8S46L8AFF47Gie3S0+vSjyexZg5OwB47rpoIqp7lXuYIiIipaWPH0Sk7AwGCIiEax6Du6fByP1w989wzRMQGIWJQlobE3jWeQqzXZ/ns2MDSf/xQd56+w1+X/sXBYVme9+BiIjdfPzxx9SpUwc3NzfatGnD2rVrL3rsL7/8QlxcHH5+fnh6etKsWTP+97//VWC0DqggF34dAvGjMWAhMfBaLE0HXPKUQrOFZ6ZsIiffTLvIQO5tW6diYhURESkljdwREdtzdreuQFKvu7VOz6n9sGcB7JmPZd9iquencYtpGZxZhnn2O+yYG0VhRHeiO/bDJTwOjFqFRESqhsmTJzN8+HA++eQT2rRpw4QJE+jVqxc7d+4kKCjoguMDAgJ44YUXiI6OxsXFhVmzZnH//fcTFBREr16XX/Wpysk6DpPvhoOrwGCksMfrbEmtQU3DpT/f/HTpXjYePI23qxNv3xarUaYiIlLpKbkjIuXPvw60ehBaPYihIA+SVpO7Yx4Zf82lWtYeGpl3wZ5dsOf/yHH2wymqG071e0C9buB14R83IiJXi/Hjx/Pwww9z//33A/DJJ58we/ZsvvrqK5577rkLju/SpUux50OHDuXbb79l+fLlF03u5ObmkpubW/Q8PT0dgPz8fPLz8210J5VQ6g6cpgzEkHYQi6sPhf2/IK9WR4iPv+R9JyRn8F78LgBeuL4BQZ5OV3c7lcLZ9lC7lJ7a0DbUjmWnNrSN8mrHK7mekjsiUrGcXKBuJ1zrdsL1+jc4c+Ig6+dPIy/hD1qZN+OTfxq2/2zdAEJj/xkFVLM1mPRjS0SuDnl5eWzYsIFRo0YV7TMajXTv3p1Vq1Zd9nyLxcLChQvZuXMnY8eOvehxY8aM4dVXX71g/x9//IGHh0fpgq/kgtP+JG7/RAzmHDJdglgTMZzMnTmwMx6A+Pj4fz2vwAzjt5rILzTQ2N+M29HN/P775ooM3aFcrB2l5NSGtqF2LDu1oW3Yuh2zs7NLfKz+ShIRu3IPrEXHAcPJLRjKjPUHWLZ4Dg0y19LZuJmmxkQ4utm6LXsXXH0hovM/yR7fGvYOX0Sk1I4fP05hYSHBwcHF9gcHB5OQkHDR89LS0qhRowa5ubmYTCb+7//+jx49elz0+FGjRjF8+PCi5+np6YSHh9OzZ098fC5dVNjhWCwY13yM8c8JGLBgrt0e15u/ppNHAGD9BDQ+Pp4ePXrg7Ox8wenj43dzODsRfw9nPn+kHdW8XCv6DhzC5dpRLk9taBtqx7JTG9pGebXj2dG2JaHkjohUCq5OJgZcE8EtrR7n9239eXbRHo4nH6KjcQvXOm2hm/M2PHPTYMdM6wYQ1NA6dated6jVFpzUCReRq5+3tzebNm0iMzOTBQsWMHz4cCIiIi6YsnWWq6srrq4X/nx0dna+ujryBbkwezhs+t76vOV9GK9/B6Ppwnv8t3vfePAUny5LBODN/k0I9dfqWJdz1f0bsgO1oW2oHctObWgbtm7HK7mWkjsiUqk4mYz0jQ2jT9NQFiak8tGi2jx1sCPGPDOxpkQeDdtHF+Nm3FL+hNTt1m3lh+DsCXU7QdTfo3r869j7VkRELqlatWqYTCZSUlKK7U9JSSEkJOSi5xmNRurVqwdAs2bN2LFjB2PGjLlocqdKOK9wMr3GQJtHras5lsCZvEJGTNmM2QL9moXRu0loOQcsIiJiW0ruiEilZDAY6BYTTNfoIFbvO8n/Ld7Dst1GHkuKxGDowc0N3Bla9zC1Tq6EPfMhKxV2zbFuAIH1oF4Pa6KnTnvrCl4iIpWIi4sLLVu2ZMGCBfTr1w8As9nMggULGDJkSImvYzabixVMrnJStsOPA+D0Qev03du+sv7svwJj5yaw73gWIT5uvNq3cTkFKiIiUn6U3BGRSs1gMNA2MpC2kYFsTjrN/y3ew7y/Uvg54Qw/JwTQMeo+htz8Gq3dD2PYu8C65PrB1XBij3VbMxGc3Kyjehr2g+gbwN3P3rclIgLA8OHDGTRoEHFxcbRu3ZoJEyaQlZVVtHrWvffeS40aNRgzZgxgLY4cFxdHZGQkubm5/P777/zvf/9j4sSJ9rwN+9k5F35+EPIywb8u3DUFqte/okus2HOcb1buB2DsrU3x9dC0BBERcTxK7oiIw4gN9+PTe+LYlZLBxMV7mbn5CMt2H2fZ7uPE1fZn8LX30KXDcAy56bBviXVEz575kH4Ydv9h3X5zttbpadQfGlwPbldZMVERcSgDBgzg2LFjvPTSSyQnJ9OsWTPmzp1bVGT54MGDGI3GouOzsrJ44oknOHToEO7u7kRHR/P9998zYMAAe92CfVgs1im58S8BFqjTEW7/Dv4unFxS6Tn5/GeqdTWsgW1q0bl+9XIIVkREpPwpuSMiDqd+sDfvDWjG093r8+nSvUxdf4j1B05x/zfriAn1YfC1kfRu3AdTw77WPwBSd0DCLPhrurVGz6651s3kah263/hmqN8LXL3tfWsiUgUNGTLkotOwFi9eXOz566+/zuuvv14BUVViBbkwq3jhZK5/B/6lcPLlvPbbdo6k5VArwIPnr4+xbZwiIiIVSMkdEXFYtQI9eKN/E4Z2i+KL5Yl8v/oAO46mM2TSn9SttovHO0fSr3kNXIIbQnBD6PysNdHz1wz46xc4vgt2zrZuTm4Q1dM6oqd+L3DxtPftiYjI+cpYOPlc8dtTmLbhEAYDvHNbLJ6u6haLiIjj0m8xEXF4QT5uPH99DE90ieSblfv5esV+Eo9n8ezPW3hv/i4e6RTBHa1q4e5igqAY69blOesonm2/WBM9J/f9s8y6s4c1wdPoZojqoWLMIiKVgQ0KJ591IiuPUb9sAeDhjhG0rntl07lEREQqG+PlDyk/EydOpGnTpvj4+ODj40Pbtm2ZM2eOPUMSEQfm5+HCsO71WfFcV164Pobq3q4cTcvh1d+2037sQt6L38XxzL9XlDEYILgRdBsNT26ER5dCh6fBrzbkZ1uncE25B96uB9MehITZkJ9j3xsUEamqds6FL3tYEzv+deGh+aVO7Fgs8PLM7RzPzKN+sBfDe1xZAWYREZHKyK4jd2rWrMlbb71FVFQUFouFb7/9lptuuok///yTRo0a2TM0EXFgXq5OPNwpgnva1ubnjYf4ZMlekk6e4f0Fu/lkyV5ublGThzrWJbK6l/UEgwFCY61bt5fhyJ/W5M5fMyDtIGybZt1cfaxFmBv1h8iu4ORi1/sUEbnq2ahw8rk2HDcwb08qTkYD429vhpuzyXbxioiI2Ildkzt9+vQp9vyNN95g4sSJrF69WskdESkzN2cTA9vUZkBcOHP/SubzpfvYfCiNH9ce5Me1B+keE8zDHevSum4AhrP1GgwGqNHCuvV4DQ5vsE7d2j7DuurWlp+sm5svRN9onboV0blUhTxFROQSbFg4+azk9BymJVoHrj/ZNYrGNXxtEKiIiIj9VZqaO4WFhUydOpWsrCzatm37r8fk5uaSm5tb9Dw9PR2A/Px88vPzKyTOyuLs/Va1+7Y1tWPZOUob9oqpTs/oaqw7cIqvVhxgQcIx5u9IYf6OFJrW8OHB9nXo2TAIJ9N5s1WDY61b15cxHF6PYfsMjDt+xZCZApt+gE0/YHH3x9LgBswx/bDU6QDGK//R6ijtWJmpDW2jvNpR3xe5IjYsnHyu0b9u50yhgSY1fHji2kgbBSsiImJ/dk/ubN26lbZt25KTk4OXlxfTp0+nYcOG/3rsmDFjePXVVy/Y/8cff+Dh4VHeoVZK8fHx9g7hqqB2LDtHasO+/tCmGSw+YmTtMQNbDqczdMoWAlwtdA41c02QBbeLjtLvAPXaEZi1i7BTawk7vQ63M6cwbPoe46bvyXXy5ohvHEf823DcK9r6R8kVcKR2rKzUhrZh63bMzs626fXkKmbDwsnnWrn3OIt3HcdksDDu5sY4n5/MFxERcWB2T+40aNCATZs2kZaWxrRp0xg0aBBLliz51wTPqFGjGD58eNHz9PR0wsPD6dmzJz4+PhUZtt3l5+cTHx9Pjx49cHbWdJDSUjuWnSO34f3AicxcflibxPdrkjiZnc/0/SYWJDtxZ6tw7rkmnGAft0tfxFxIwcGV1hE9O2fhmn2CuicWUffEIiye1TFH98EScxOW8GvAePG6Do7cjpWF2tA2yqsdz462FbmknXPh5wchL9NaOPmuKVC97AWPLRYL4+buBKBdkIV6QV5lvqaIiEhlYvfkjouLC/Xq1QOgZcuWrFu3jvfff59PP/30gmNdXV1xdXW9YL+zs3OV7chX5Xu3JbVj2TlqG4b4O/NMrxieuLY+P288xJfLE0k8nsWnyxL5auV++sbW4OFOdYkOuVgC2Rmiulq3wvGwf6m1GPOO3zBkHcO04SvY8BV4hUDDm6DxzVCzNRj//RNjR23HykRtaBu2bkd9T+SSyqFw8rnit6ewKek07s5GetYssMk1RUREKhO7J3fOZzabi9XVERGpCO4uJu6+pjZ3ta7F/B0pfL5sH+v2n+LnjYf4eeMhOkZV45FOEXSoV+2f4svnMzlZV9GK7Ao3jId9S6yJnoTfIDMZ1n5q3XxqQMN+1lW3asaVuYaEiIhDK4fCyecqNFt4e5511M59bWvjk7/bJtcVERGpTOya3Bk1ahS9e/emVq1aZGRkMGnSJBYvXsy8efPsGZaIVGFGo4GejULo2SiEPw+e4otliczZdpRlu4+zbPdxokO8ebhjBH1iw3BxukS9BpMzRHW3bgXvwb5F1lW3EmZbV91a/bF18w2HRv0wNOhr/eRaRKQqKafCyeea8edhdqdm4uvuzEMd6rB8kZI7IiJy9bFrcic1NZV7772Xo0eP4uvrS9OmTZk3bx49evSwZ1giIgA0r+XPxwP9STqZzZfLE5myPomE5AyembqZcfMSuL99Xe5sXQtf98t8uuzkAvV7Wbf8HNi7wDqiZ+ccSEuClR/itPJDrjN5YTr9BYQ0hZDGENwYqjcApwuno4qIOLxyKpx8rtyCQt6bvwuAx7tE4nO5n9ciIiIOyq7JnS+//NKeby8iUiLhAR680rcRw7pH8cOag3y7cj8p6bm8NSeBDxfsZkCrWtzfvg7hASVYtc/ZDaJvsG75Z2B3PPw1HcuuubjmZ8L+ZdbtLKMTVGvwd7KnkTXhE9IEvILK74ZFRMpbORVOPt+Paw5y6NQZgrxdGdS2DmC2+XuIiIhUBpWu5o6ISGXl5+HC4Gvr8VDHuszcdITPl+1jV0omX61I5NtV++ndOIRHOkXQtKZfyS7o7A4N+0LDvhScyWDFjC/pUM8Pp+M7IHkbpGyFnDRI/cu6ncsz6J/RPSFNrF+rRdmsRoWISLko58LJ58rKLeCjRXsAeKpbFO4uJvLzldwREZGrk5I7IiJXyNXJxG1x4dzasiZLdx/n86X7WL7nOLO2HGXWlqO0qRvAwx0j6BodhNFYwroRTm6kedTF0ux6OLuqkMUCaYcgZds/yZ7kbXByH2Slwt6F1u0skwtUj/4n2XM2+VMOfzSJiFyxci6cfL6vVyRyPDOP2oEeDGgVXi7vISIiUlkouSMiUkoGg4HO9avTuX51/jqSxpfLEpm5+QhrEk+yJvEkEdU9ebhjBP2b18DN2VSaNwC/cOvWoPc/+/OyrLUqUradk/j5C/IyIHmLdTuXT43iyZ6QJhAQAcZSxCQiUhoVUDj5XKey8vh0yT4Ahveoj7PpEgXwRURErgJK7oiI2ECjMF/GD2jGf65rwDcr9jNpzUH2Hcti1C9befePndxzTR3uaVubAE+Xsr+ZiyeEt7JuZ5nNcPrAOcmebZC81bov/bB1233OSoTOHhAUU3xaV3AjcPMpe3wiIueqgMLJ5/tkyV4ycguICfWhT9Owcn0vERGRykDJHRERGwr1dWfU9TEM6VqPyeuS+HrFfg6fPsN783cxcckebm1Zkwc7RFC3mqdt39hohIC61i2mzz/7c9L+GeWTvPXv0T7bIT8bDm+wbufyq33eKJ/G4FfHen0RkStVQYWTz5WclsM3K/cD8J9e9Us+PVZERMSBKbkjIlIOvN2ceahjBIPa1eH3rUf5fNk+th1O5/vVB/lhzUF6xATzSKcIWtb2x1BO0xIAcPOF2m2t21nmQmvdnrPJnrMjfdIPW0f6nD4AO2f/c7yLNwQ3tCZ7QmMhrBkENVTxZhG5uAosnHy+DxbuJrfATFxtf65toJUFRUSkalByR0SkHDmbjNzUrAZ9Y8NYve8kXyzbx4KEVP7YnsIf21NoFu7HI50i6Fo/sOKCMpqsK2tVi4LGN/+zP/vkhdO6jiVYa/kkrbFuZ5lcrdO4wppBaDMlfETkHxVcOPlcicezmLwuCYCRvaPLN3kuIiJSiSi5IyJSAQwGA20jA2kbGcie1Ay+WJbILxsPsynpNE/8sJFwf3da+hi4JiuPYD87JUg8AqBuJ+t2VmE+HN/9d7JnCxzdDEc2Q24aHNlo3c5SwkdEKrhw8vnGx++i0Gzh2gbVaVVHKwWKiEjVoeSOiEgFqxfkzVu3NOWZng3436r9fLf6AEmnzpB0ysTst5fQq1EId7SqRbvIQPvXijA5/z0lqyE0vd26z2KBU4lwZBMc+ROObrImfXKU8BGp0uxQOPlcfx1J47fNRwAY0atBhb2viIhIZaDkjoiInVT3dmV4zwY83qUeU9Yd4POF2zmUBbO2HGXWlqPU9HdnQFw4t8bVJNTX3d7h/sNgsC6lHhDxz7SuK0r4uFjr95yb8KkeA042WElMROzDbIZpD1gTOxVUOPl878zbCUCf2DAahflW6HuLiIjYm5I7IiJ25u5i4q7W4fgd30rtZh34+c+jzNh0mEOnzvBu/C7em7+LLg2CGNAqnK7RQTibKuHKVZdL+Bzd9HfSRwkfkauS0Qg3fwaL3oB+EyukcPK51iaeZNHOYzgZDTzTo2KTSiIiIpWBkjsiIpVIozAfmtUO5PnrY5iz7Sg/rUtibeJJFiaksjAhlWpertzasiYDWoXbfjl1WytRwufvr0r4iDi+0KZw1+QKf1uLxcK4uQkA3N4qnDqV/WejiIhIOVByR0SkEnJ3MXFzi5rc3KIme49lMmV9Ej9vOMTxzFw+WbKXT5bspU3dAO5oHU7vxqG4OZvsHXLJKOEjIja2aGcq6w+cwtXJyFNdo+wdjoiIiF0ouSMiUslFVvdiVO8YRvRswIIdqUxed5Alu46xJvEkaxJP8tKvf9G/eQ0GtAp3zDoTNkn4NMIYEkut40ZIDoewJiraLFIFmM0Wxs211tq5r10dQnzd7ByRiIiIfSi5IyLiIJxNRq5rHMJ1jUM4cvoM0zYcYvK6JA6fPsN3qw7w3aoDNKnhy4BW4fRtFoaPmwMnNy6a8Nn/T8HmYgmfPzEd+ZPmAF9+BU5uENIUwppDjRbWr4FR1rogInLV+G3LERKSM/B2c+LxLpH2DkdERMRulNwREXFAYX7uPNUtiiHX1mPF3uP8tC6J+L9S2Ho4ja2H03h99nZuaBLGHa3Diavtj8Fg5yXVbcFggIC61u38hM/RTRQe2sDJrQuoln8IQ246HFpr3c5y8YbQWKjR3JrsCWsB/nWs1xURh5NfaGZ8/C4AHu0UgZ+HpmeKiEjVpeSOiIgDMxoNdIyqTseo6pzMymP6n4f5ae1Bdqdm8vPGQ/y88RAR1T25o1U4N7eoSTUvV3uHbFvnJHzM9W9kZU4rru99Hc4Zh+DwRusonyMbrat05WXAgeXW7Sx3/38SPWdH+XiHKuEj4gAmr0viwIlsqnm5cH/7uvYOR0RExK6U3BERuUoEeLrwYIe6PNC+DhsPnmbKuiR+23KEfceyePP3BMbN3UmPhsEMaBVOx6jqmIxXaQLDYITASOvW9DbrvsICOL7r73o9f1oTPynb4Mwp2LvQup3lFVw82RPWHDyr2edeRORfnckr5IMFuwF4smsUnq7q0oqISNWm34QiIlcZg8FAy9r+tKztz+g+DZm1+Qg/rUtiU9Jp5mxLZs62ZMJ83bgtLpzb4mpS09/D3iGXP5MTBDe0bs3vtu4ryIPUv84Z4fMnpO6AzBTYNce6neVbq/h0rrBm4OaAxatFrhLfrNxPakYuNf3dubN1LXuHIyIiYndK7oiIXMW8XJ24o3Ut7mhdi4TkdCavS2L6n4c5kpbD+wt288HC3XSMqs4drcLpHhOMi1MVKjjs5PJ3sqb5P/vysiF5a/ERPid2Q9pB67b913+ODaxXfIRPSBNw8az4+xCpYtLO5PPJkr0APN29ftX6uSUiInIRSu6IiFQR0SE+vNynESOvi+aP7SlMXneQFXtOsHTXMZbuOkaApwu3tLAuqV4vyNve4dqHiwfUamPdzspJs9bsObeGz+mDcGKPdds6xXqcwQjVY/5O9vydNApuDE5XWZ0jETv7bOle0s7kUz/Yi37Na9g7HBERkUpByR0RkSrGzdlE39gw+saGcfBENlPWJzF1QxIp6bl8viyRz5clElfbn9tbhXNj01A8XKr4rwo3X6jbybqdlXXin0TP2RE+mcnWaV6pf8Gm763HGZ0hpHHxos3Vo63TxETkiqVm5PDV8v0AjOjZ4OqtHSYiInKF1LsUEanCagV6MKJXA4Z1j2LJrmP8tC6JhQmprD9wivUHTvHab9vpExvGHa3CaVrT9+pYUt0WPAMhqrt1Oyv9aPFkz5GN1oLNZ+v58JX1OJOLdQn2gL+LPgdE/P01EnxqgFFTTEQu5qOFeziTX0jzWn70aBhs73BEREQqDSV3REQEJ5ORbjHBdIsJJjU9h2kbDxUtM/zj2oP8uPYg0SHe3Nm6Fv2a18DX3dneIVc+PqHgcwNE32B9brHA6QPFCzYf2WRdkv34Lut2PpPr30u7R0JgxDkJoEjrEu1K/EgVlnTS+vMI4D+9GijZLCIicg4ld0REpJggHzee6FKPxzpFsibxJJPXHeT3bckkJGfw8sy/GDNnBzc2DeOuNrVoHu6nP7AuxmCwjtDxrwONb7buM5shLQlO7oUTe+Hkvr+/7oVT+6EwF44lWLfzObn/Pconwvq1WOInxPp+Ilex9+J3kV9ooWNUNdpFVrN3OCIiIpWKkjsiIvKvjEYDbSMDaRsZyKvZ+Uz/8xCT1h5kV0om0zYcYtqGQ0SHeHNXm1rc1EyjeUrEaAT/2tYtsmvx1woLzkn87DsnAbQXTh2AgjP/1PQ5n7PnOYmfc5I+ARHgFaTEjzi8hOR0pm86DFhH7YiIiEhxSu6IiMhl+Xo4c1/7ugxqV4eNB08xaU0Ss7YcISE5g5d+/Ys3f9donjIzOf09Jasu1DvvtcL8v1fo+jvZc+6In9MHIT8LUrZat/O5eFuveTbhc+5Xj0AlfsQhvDNvFxYLXN8khKY1/ewdjoiISKWj5I6IiJSYwWCgZe0AWtYO4KUbG2o0T0UxOVuTMYGRF75WkGet7XM22VP0dZ91JFBeBiRvsW7nc/W9YJqXwbc2zgUZ5X9PIiW04cAp5u9IwWiA4T00akdEROTfKLkjIiKlotE8lYSTC1SLsm7nK8i11vL5t8RP+iHITTtnNa+/LwdcD+T37gvOSs6JfVksFsbNtdagurVlTeoFedk5IhERkcpJyR0RESkTjeapxJxcoXoD63a+/DNwMtE6xeucAs+WE3s4k5OHs7N7xccrcp6lu4+zJvEkLk5Ghnavb+9wREREKi0ld0RExGY0mseBOLtDcEPrdo6C/Hzmz5pJbzuFJXKW2Wzh7XnWUTv3XFObGn5KOIqIiFyMkjsiImJzGs3j2CxGdQ/E/uZsS2bb4XQ8XUw80eVf6k2JiIhIEaO9AxARkavb2dE884Z14ufH23JLi5q4OhmLRvO0eXM+/5m6mY0HT2GxWOwdrohUAgWFZt79YycAD3WMINDL1c4RiYiIVG76aE5ERCrEpUbzTN1wiKkazSMif5u24RD7jmcR4OnCQx3r2jscERGRSk/JHRERqXAlqc3Tp2kYd6o2j0iVk5NfyPsLdgPwRJdIvN2U6BUROVdhYSH5+fk2uVZ+fj5OTk7k5ORQWFhok2tWRaVtR2dnZ0wmk01iUHJHRETsRqN5ROR8368+wNG0HMJ83bj7mtr2DkdEpNKwWCwkJydz+vRpm14zJCSEpKQkfZhWBmVpRz8/P0JCQsrc/kruiIhIpaDRPCKSkZPPx4v2ADCse33cnG3zaaaIyNXgbGInKCgIDw8Pm/SFzGYzmZmZeHl5YTSqJG9plaYdLRYL2dnZpKamAhAaGlqmGJTcERGRSkWjeUSqrs+XJXIqO5/I6p7c3KKGvcMREak0CgsLixI7gYGBNruu2WwmLy8PNzc3JXfKoLTt6O7uDkBqaipBQUFlmqKl5I6IiFRa54/m+WHNQWZvOfqvo3kah3jaO1wRKYMTmbl8uWwfAM/0bICTSX9kiIicdbbGjoeHh50jEVs7+z3Nz89XckdERK5u547mefnGRv8+mifYi8YeBtqfyaeas0bziDiajxftJSuvkCY1fOndOMTe4YiIVEqaln71sdX3VMkdERFxKBcdzZOSSQImZo5bQs+GwdwWF06HetUwGdUJEqnsDp3K5vvVBwB49roG+uNFRETkCmm8q4iIOKSzo3nG396Mtc93Z/QN0YR6WMgrMDNry1EGfbWWDmMX8va8BBKPZ9k7XBG5hPfn7yav0EzbiEA61Ktm73BERKSSq1OnDhMmTCjx8YsXL8ZgMNh0pbHKRiN3RETE4fl6OHPvNbUIPLGNOs07MH3TUX7dfISjaTl8vGgvHy/aS1xtf26Lq8kNTcPwctWvP5HKYk9qBj9vPATAfzRqR0TkqnK5n+kvv/wyr7zyyhVfd926dXh6lrzeYrt27Th69Ci+vr5X/F6OQr1bERG5ahgM0CjMh2a1A3n+hhjmb09l6oYklu46xvoDp1h/4BSvzNxO7yYh3NYynDZ1AzBq2paIXb37xy7MFujRMJgWtfztHY6IiNjQ0aNHix5PnjyZl156iZ07dxbt8/LyKnpssVgoLCzEyenyaYrq1atfURwuLi6EhFzd9dw0LUtERK5Krk4mbmgayjf3t2blc90YeV00EdU9OZNfyC8bD3Pn56vp/M4i3p+/m0Onsu0drkiVtDnpNHO2JWMwwH96NbB3OCIiDsVisZCdV1Dm7Uxe4RWfY7FYShRjSEhI0ebr64vBYCh6npCQgLe3N3PmzKFly5a4urqyfPly9u7dy0033URwcDBeXl60atWK+fPnF7vu+dOyDAYDX3zxBf3798fDw4OoqChmzpxZ9Pr507K++eYb/Pz8mDdvHjExMXh5eXHdddcVS0YVFBTw1FNP4efnR2BgICNHjmTQoEH069ev1N+z8qSROyIictUL8XXj8S6RPNY5go0HTzNtQxK/bT5K0skzvDd/FxMW7KJdZCC3tqzJdY1CcXcp/TKUIlJyb8+zfnrbv3kN6gd72zkaERHHcia/kIYvzbPLe29/rRceLrZJJzz33HO88847RERE4O/vT1JSEtdffz1vvPEGrq6ufPfdd/Tp04edO3dSq1ati17n1VdfZdy4cbz99tt8+OGHDBw4kAMHDhAQEPCvx2dnZ/POO+/wv//9D6PRyN13382IESP44YcfABg7diw//PADX3/9NTExMbz//vvMmDGDa6+91ib3bWsauSMiIlWGtQizP2Nubsq6F7rz3oBY2kUGYrHAij0neHryZlq/MZ9Rv2xhw4FTJf5USkSu3Io9x1m+5zjOJgNPd69v73BERMROXnvtNXr06EFkZCQBAQHExsby6KOP0rhxY6Kiovjvf/9LZGRksZE4/+a+++7jzjvvpF69erz55ptkZmaydu3aix6fn5/PJ598QlxcHC1atGDIkCEsWLCg6PUPP/yQUaNG0b9/f6Kjo/noo4/w8/Oz1W3bnEbuiIhIleTuYqJ/85r0b16TpJPZ/LzxENM2HOLQqTP8uDaJH9cmEVndk1tbhnNzixoE+7jZO2SRq4bFYmHc36N27mpdi/AADztHJCLieNydTWx/rVeZrmE2m8lIz8DbxxujseRjP9ydbTfKOS4urtjzzMxMXnnlFWbPns3Ro0cpKCjgzJkzHDx48JLXadq0adFjT09PfHx8SE1NvejxHh4eREZGFj0PDQ0tOj4tLY2UlBRat25d9LrJZKJly5aYzeYrur+KouSOiIhUeeEBHgzrXp+nukaxJvEkUzckMWdrMnuPZTF2bgJvz0ugU/3q3NYynO4Ng3B10rQtkbKY91cKm5NO4+FiYkjXKHuHIyLikAwGQ5mnRpnNZgpcTHi4OF1RcseWzl/1asSIEcTHx/POO+9Qr1493N3dufXWW8nLy7vkdZydnYs9NxgMl0zE/NvxjjxqW8kdERGRvxmNBtpGBtI2MpBX++bz+9ajTF1/iPUHTrF45zEW7zyGn4czN8WGcVtcOI3CfLRss8gVKjRbePcP66idB9rXpbq3q50jEhGRymTFihXcd9999O/fH7CO5Nm/f3+FxuDr60twcDDr1q2jU6dOABQWFrJx40aaNWtWobGUlJI7IiIi/8LbzZkBrWoxoFUt9h3LZNqGQ/yy8TDJ6Tl8u+oA3646QHSIN7fFhdOvWRiBXvoDVaQkpv95mN2pmfi6O/Nwpwh7hyMiIpVMVFQUv/zyC3369MFgMDB69Gi7TIV68sknGTNmDPXq1SM6OpoPP/yQU6dOVdoP9lRQWURE5DIiqnvx7HXRrHiuK98+0Jobm4bi4mQkITmD/87aTps3F/Do/9YTvz2F/MLKOQ9bpDLILSjkvfhdADzeJRJfd+fLnCEiIlXN+PHj8ff3p127dvTp04devXrRokWLCo9j5MiR3Hnnndx77720bdsWLy8vevXqhZtb5azDqJE7IiIiJWQyGuhcvzqd61cnLTufmZsPM3XDIbYcSmPeXynM+yuFal6u9G9unbalpZ1Fipu05iCHT58h2MeVQW3r2DscERGpQPfddx/33Xdf0fMuXbr8a42bOnXqsHDhwmL7Bg8eXOz5+dO0/u06p0+fvuh7nR8LQL9+/Yod4+TkxIcffsiHH34IWOsTxcTEcPvtt//r/dmbkjsiIiKl4OvhzD1t63BP2zrsTM5g6vokZmw6zPHMXD5flsjnyxKJrenLrXHh9G0ahq+HRihI1ZaVW8BHC/cA8FS3KNxdVJhcREQqrwMHDvDHH3/QuXNncnNz+eijj0hMTOSuu+6yd2j/SskdERGRMmoQ4s2LNzZkZO9oFu88xtT1SSxMSGXzoTQ2H0rjv7O207NhMLfFhdOhXjVMxso5V1ukPH21PJETWXnUCfTg9rhwe4cjIiJySUajkW+++YYRI0ZgsVho3Lgx8+fPJyYmxt6h/Ssld0RERGzE2WSkR8NgejQM5nhmLjP+PMy0DYdISM5g1pajzNpylBAfN25sGkrfZmE0qeFbaYvyidjSqaw8Plu6D4Cne9TH2aSyjyIiUrmFh4ezYsUKe4dRYkruiIiIlINqXq481DGCBzvUZdvhdKZuSOLXTUdITs/hi+WJfLE8kTqBHvSJDaNvbBhRqs8jV7GJS/aSkVtATKgPfZqG2TscERGRq46SOyIiIuXIYDDQpKYvTWr68sINMSzeeYzfNh9h/o4U9p/I5sOFe/hw4R6iQ7zp2yyMPk3DCA/wsHfYIjaTnJbDtyv3A/BsrwYYNS1RRETE5pTcERERqSCuTiZ6NQqhV6MQsnILmL8jhZmbjrB09zESkjNImLuTcXN30ryWH31jw7ihaShB3pVzuU2Rknp/wW5yC8y0quNPlwbV7R2OiIjIVUkTnkVEROzA09WJm5rV4Mv7WrHuhe68dXMT2kUGYjDAnwdP8+pv27nmzQXc9flqflp7kLTsfHuHLOXk448/pk6dOri5udGmTRvWrl170WM///xzOnbsiL+/P/7+/nTv3v2Sx9vbvmOZTFmfBMCz10WrxpSIiEg5UXJHRETEzvw8XLijdS0mPXwNa0Z146UbG9K8lh9mC6zce4LnftlK3BvxPPTtOn7ddJis3AJ7hyw2MnnyZIYPH87LL7/Mxo0biY2NpVevXqSmpv7r8YsXL+bOO+9k0aJFrFq1ivDwcHr27Mnhw4crOPKSGR+/i0Kzha7RQbSqE2DvcERERK5ampYlIiJSiQT5uPFAh7o80KEuSSezmbn5CL9tPkJCcgbzd6Qyf0cq7s4musUE0Tc2jM4NquPqZLJ32FJK48eP5+GHH+b+++8H4JNPPmH27Nl89dVXPPfccxcc/8MPPxR7/sUXX/Dzzz+zYMEC7r333gqJuaS2HU5j1pajAIzo2cDO0YiIiFzdlNwRERGppMIDPBh8bT0GX1uPXSkZ/Lb5CDM3H+HAieyipdW93Zy4rlEIfZuF0TYiECctMe0w8vLy2LBhA6NGjSraZzQa6d69O6tWrSrRNbKzs8nPzycg4OKjYnJzc8nNzS16np6eDkB+fj75+eU33W/c3B0A3NgkhKjq7uX6XiV1NobKEIsjUzuWndrQNqpSO+bn52OxWDCbzZjNZptd12KxFH215XVtqWvXrsTGxvLee+8BEBERwdChQxk6dOhFzzGZTPz888/069evTO9d0uuUpR3NZjMWi4X8/HxMpuIf2F3Jv227JnfGjBnDL7/8QkJCAu7u7rRr146xY8fSoIE+3RERETlX/WBvnunZgOE96rPlUBq/bT7CrC1HSU7PYeqGQ0zdcIhqXi5c3ySUvrFhtKjlr1WJKrnjx49TWFhIcHBwsf3BwcEkJCSU6BojR44kLCyM7t27X/SYMWPG8Oqrr16w/48//sDDo3xWZtuTDkt3O2E0WGhmOsTvvx8ql/cprfj4eHuHcFVQO5ad2tA2qkI7Ojk5ERISQmZmJnl5eTa/fkZGhs2vCXDHHXdQUFDAtGnTLnht5cqV3HDDDSxbtozGjRtf9BoFBQXk5eUVfTgxf/58PDw8ip5fzJkzZy57zFlvvfUWs2fPZtmyZcX2JyQk4OfnV+LrlKYd8/LyOHPmDEuXLqWgoPjU++zs7BJfx67JnSVLljB48GBatWpFQUEBzz//PD179mT79u14enraMzQREZFKyWAwEBvuR2y4H89fH8Pa/SeZufkIc7Ye5XhmHt+tOsB3qw5Qw8+dG2OtiZ6GoT4qZHsVeuutt/jpp59YvHgxbm4XX1Vt1KhRDB8+vOh5enp6Ua0eHx8fm8dlsVi444t1wGkGxIUzqG9Dm79HaeXn5xMfH0+PHj1wdna2dzgOS+1YdmpD26hK7ZiTk0NSUhJeXl6X/Jl/pSwWCxkZGXh7e5dLX+GRRx7htttuIz09nZo1axZ7berUqcTFxdGuXbtLXsPJyQkXF5ei31kl/d3l7u5e4mNdXV0xmUwXHF/S88vSjjk5Obi7u9OpU6cLvrclTSqBnZM7c+fOLfb8m2++ISgoiA0bNtCpUyc7RSUiIuIYjEYD10QEck1EIK/2bcTyPcf5bdMR5v2VzOHTZ/h0yT4+XbKPyOqe9IkNo29sGBHVvewdtvytWrVqmEwmUlJSiu1PSUkhJCTkkue+8847vPXWW8yfP5+mTZte8lhXV1dcXV0v2O/s7Fwufwwt2JHCxoOncXM2MqxHg0r5B1d53XtVo3YsO7WhbVSFdiwsLMRgMGA0GjEa/56CbbFAfslHdvwbs9kM+dkY8k3/XLcknD2gBEmMvn37Ur16db777jtefPHFov2ZmZlMmzaN5557joEDB7J06VJOnTpFZGQkzz//PHfeeWex65y9d4A6deowbNgwhg0bBsDu3bt58MEHWbt2LREREbz//vsAxdpq5MiRTJ8+nUOHDhESEsLAgQN56aWXcHZ25ptvvuG1114DKJoW9fXXX3PfffdhMBiYPn160bSsrVu3MnToUFatWoWHhwe33HIL48ePLxoJ+8ADD5CWlkaHDh149913ycvL44477mDChAkX/TdqNBoxGAz/+u/4Sv5dV6qaO2lpaQAXnTdurznjlVFVml9antSOZac2tA21Y9mpDaFDhD8dIvx5tU80i3YeY/bWZBbtOs7eY1lMmL+bCfN30zDUmxubhnBjk1BCfS/85K+82rEqf18uxsXFhZYtW7JgwYKiTqPZbGbBggUMGTLkoueNGzeON954g3nz5hEXF1dB0ZaM2Wzh7Xk7ARjUrg7BPrb7dFlERM6Tnw1vhpXpEkbArzQnPn8EXC4/28bJyYl7772Xb775hhdeeKFoVMvUqVMpLCzk7rvvZurUqYwcORIfHx9mz57NPffcQ2RkJK1bt77s9c1mMzfffDPBwcGsWbOGtLS0oqTPuby9vfnmm28ICwtj69atPPzww3h7e/Pss88yYMAAtm3bxty5c5k/fz4Avr6+F1wjKyuLXr160bZtW9atW0dqaioPPfQQQ4YM4auvvio6btGiRYSGhrJo0SL27NnDgAEDaNasGQ8//PBl76csKk1yx2w2M2zYMNq3b3/R+Xb2mDNe2VWF+aUVQe1YdmpD21A7lp3a8B/X+0LX5rDllIGNxw3sPG1g+9EMth/NYNy83UR4W2hRzUyzQAve530wZOt2vJI541XJ8OHDGTRoEHFxcbRu3ZoJEyaQlZVVtHrWvffeS40aNRgzZgwAY8eO5aWXXmLSpEnUqVOH5ORkALy8vPDysv+orJl/r+zm7ebE450j7R2OiIhUAg888ABvv/02S5YsoUuXLoB1ZMwtt9xC7dq1GTFiRNGxTz75JPPmzWPKlCklSu7Mnz+fhIQE5s2bR1iYNdH15ptv0rt372LHnTtqqE6dOowYMYKffvqJZ599Fnd3d7y8vIrqGl3MpEmTyMnJ4bvvvisqI/PRRx/Rp08fxowZg7u7OwD+/v589NFHmEwmoqOjueGGG1iwYEHVSe4MHjyYbdu2sXz58oseU9FzxiuzqjS/tDypHctObWgbaseyUxte3M1/fz2Zlcfcv1KYtTWZ9QdOsS/DwL4ME9MPGGgbEcCNTUK4NiqA1csW2bwdr2TOeFUyYMAAjh07xksvvURycjLNmjVj7ty5RUWWDx48WGyY/MSJE8nLy+PWW28tdp2XX36ZV155pSJDv0BegZnx8bsAeKxzJH4eLnaNR0TkqufsYR1BUwZms5n0jAx8vL2vfFpWCUVHR9OuXTu++uorunTpwp49e1i2bBmvvfYahYWFvPnmm0yZMoXDhw+Tl5dHbm5uiQdv7Nixg/Dw8KLEDkDbtm0vOG7y5Ml88MEH7N27l8zMTAoKCq44h7Bjxw5iY2OL1Qdu3749ZrOZnTt30qxZMwAaNWpUbNWr0NBQtm7dekXvVRqVIrkzZMgQZs2axdKlSy8osnSuip4z7giq8r3bktqx7NSGtqF2LDu14cUF+zkzqH0Eg9pHcDTtDLO3HGXm5iNsOZTG8j0nWL7nBM4mA9E+Rtp0thDiYbt21Pfk4oYMGXLRaViLFy8u9nz//v3lH1ApTV6fxMGT2VTzcuX+9nXsHY6IyNXPYCjR1KhLMpvBudB6nStJ7lyhBx98kCeffJKPP/6Yr7/+msjISDp37szYsWN5//33mTBhAk2aNMHT05Nhw4bZdEWwVatWMXDgQF599VV69eqFr68vP/30E++++67N3uNc5/d5DAZDhSwzX37fvRKwWCwMGTKE6dOns3DhQurWrWvPcERERKqMUF93HuoYwcwhHVg0ogvDe9SnXpAX+YUWEjMN+LpVis9/xEFYLBamrEsC4Mmu9fBw0b8fERH5x+23347RaGTSpEl89913PPDAAxgMBlasWMFNN93E3XffTWxsLBEREezatavE142JiSEpKYmjR48W7Vu9enWxY1auXEnt2rV54YUXiIuLIyoqigMHDhQ7xsXFhcLCwsu+1+bNm8nKyirat2LFCoxGIw0aNChxzOXFrsmdwYMH8/333zNp0iS8vb1JTk4mOTmZM2fO2DMsERGRKqVuNU+e6hZF/NOd+G1wW+6IMONksmsXQRyMwWBgyqNtebVvI+5sXcve4YiISCXj5eXFgAEDGDVqFEePHuW+++4DICoqivj4eFauXMmOHTt49NFHL1hF8lK6d+9O/fr1GTRoEJs3b2bZsmW88MILxY6Jiori4MGD/PTTT+zdu5cPPviA6dOnFzumTp06JCYmsmnTJo4fP15sIaezBg4ciJubG4MGDWLbtm0sWrSIJ598knvuuadoOrU92bXnNnHiRNLS0ujSpQuhoaFF2+TJk+0ZloiISJVkMBiIDvGmSYDF3qGIA3J3MTGoXR1cnJQYFBGRCz344IOcOnWKXr16FdXIefHFF2nRogW9evWiS5cuhISEFK0gWRJGo5Hp06dz5swZWrduzUMPPcQbb7xR7Ji+ffvy9NNPM2TIEJo1a8bKlSsZPXp0sWNuueUWrrvuOq699lqqV6/Ojz/+eMF7eXh4MG/ePE6ePEmrVq249dZb6datGx999NGVN0Y5sOuYWYtFnUcRERERERGRq13btm0vyAEEBAQwY8aMS553ufpz9evXZ9myZcX2nf8+48aNY9y4ccX2nbtkuqurK9OmTbvgvc+/TpMmTVi4cOEFx52tqfP1119fUJh6woQJFxxfHvTRioiIiIiIiIiIA1NyR0RERERERETEgSm5IyIiIiIiIiLiwJTcERERERERERFxYEruiIiIiIiIiDgALUp09bHV91TJHREREREREZFKzNnZGYDs7Gw7RyK2dvZ7evZ7XFp2XQpdRERERERERC7NZDLh5+dHamoqAB4eHhgMhjJf12w2k5eXR05OzgVLeEvJlaYdLRYL2dnZpKam4ufnh8lkKlMMSu6IiIiIiIiIVHIhISEARQkeW7BYLJw5cwZ3d3ebJIuqqrK0o5+fX9H3tiyU3BERERERERGp5AwGA6GhoQQFBZGfn2+Ta+bn57N06VI6depU5mlBVVlp29HZ2bnMI3bOUnJHRERERERExEGYTCabJQRMJhMFBQW4ubkpuVMGlaEdNalORERERERERMSBKbkjIiIiIiIiIuLAlNwREREREREREXFgDl1zx2KxAJCenm7nSCpefn4+2dnZpKena25kGagdy05taBtqx7JTG9pGebXj2d/VZ393i/2o/6SfE2Wldiw7taFtqB3LTm1oG5Wh/+TQyZ2MjAwAwsPD7RyJiIiIlERGRga+vr72DqNKU/9JRETEsZSk/2SwOPBHaGazmSNHjuDt7X3Fa8k7uvT0dMLDw0lKSsLHx8fe4TgstWPZqQ1tQ+1YdmpD2yivdrRYLGRkZBAWFobRqFnh9qT+k35OlJXasezUhrahdiw7taFtVIb+k0OP3DEajdSsWdPeYdiVj4+P/hPagNqx7NSGtqF2LDu1oW2URztqxE7loP6Tfk7Yitqx7NSGtqF2LDu1oW3Ys/+kj85ERERERERERByYkjsiIiIiIiIiIg5MyR0H5erqyssvv4yrq6u9Q3FoaseyUxvahtqx7NSGtqF2lKuZ/n3bhtqx7NSGtqF2LDu1oW1UhnZ06ILKIiIiIiIiIiJVnUbuiIiIiIiIiIg4MCV3REREREREREQcmJI7IiIiIiIiIiIOTMkdEREREREREREHpuSOgxkzZgytWrXC29uboKAg+vXrx86dO+0dlkN76623MBgMDBs2zN6hOJzDhw9z9913ExgYiLu7O02aNGH9+vX2DsthFBYWMnr0aOrWrYu7uzuRkZH897//RXXuL23p0qX06dOHsLAwDAYDM2bMKPa6xWLhpZdeIjQ0FHd3d7p3787u3bvtE2wldql2zM/PZ+TIkTRp0gRPT0/CwsK49957OXLkiP0CFikD9Z9sT/2n0lP/qWzUfyod9Z/KrrL3nZTccTBLlixh8ODBrF69mvj4ePLz8+nZsydZWVn2Ds0hrVu3jk8//ZSmTZvaOxSHc+rUKdq3b4+zszNz5sxh+/btvPvuu/j7+9s7NIcxduxYJk6cyEcffcSOHTsYO3Ys48aN48MPP7R3aJVaVlYWsbGxfPzxx//6+rhx4/jggw/45JNPWLNmDZ6envTq1YucnJwKjrRyu1Q7Zmdns3HjRkaPHs3GjRv55Zdf2LlzJ3379rVDpCJlp/6Tban/VHrqP5Wd+k+lo/5T2VX6vpNFHFpqaqoFsCxZssTeoTicjIwMS1RUlCU+Pt7SuXNny9ChQ+0dkkMZOXKkpUOHDvYOw6HdcMMNlgceeKDYvptvvtkycOBAO0XkeADL9OnTi56bzWZLSEiI5e233y7ad/r0aYurq6vlxx9/tEOEjuH8dvw3a9eutQCWAwcOVExQIuVI/afSU/+pbNR/Kjv1n8pO/aeyq4x9J43ccXBpaWkABAQE2DkSxzN48GBuuOEGunfvbu9QHNLMmTOJi4vjtttuIygoiObNm/P555/bOyyH0q5dOxYsWMCuXbsA2Lx5M8uXL6d37952jsxxJSYmkpycXOz/ta+vL23atGHVqlV2jMzxpaWlYTAY8PPzs3coImWm/lPpqf9UNuo/lZ36T7an/lP5qOi+k1OFvIuUC7PZzLBhw2jfvj2NGze2dzgO5aeffmLjxo2sW7fO3qE4rH379jFx4kSGDx/O888/z7p163jqqadwcXFh0KBB9g7PITz33HOkp6cTHR2NyWSisLCQN954g4EDB9o7NIeVnJwMQHBwcLH9wcHBRa/JlcvJyWHkyJHceeed+Pj42DsckTJR/6n01H8qO/Wfyk79J9tT/8n27NF3UnLHgQ0ePJht27axfPlye4fiUJKSkhg6dCjx8fG4ubnZOxyHZTabiYuL48033wSgefPmbNu2jU8++USdkxKaMmUKP/zwA5MmTaJRo0Zs2rSJYcOGERYWpjaUSiM/P5/bb78di8XCxIkT7R2OSJmp/1Q66j/ZhvpPZaf+k1R29uo7aVqWgxoyZAizZs1i0aJF1KxZ097hOJQNGzaQmppKixYtcHJywsnJiSVLlvDBBx/g5OREYWGhvUN0CKGhoTRs2LDYvpiYGA4ePGiniBzPf/7zH5577jnuuOMOmjRpwj333MPTTz/NmDFj7B2awwoJCQEgJSWl2P6UlJSi16TkznZODhw4QHx8vEbtiMNT/6n01H+yDfWfyk79J9tT/8l27Nl3UnLHwVgsFoYMGcL06dNZuHAhdevWtXdIDqdbt25s3bqVTZs2FW1xcXEMHDiQTZs2YTKZ7B2iQ2jfvv0Fy8ju2rWL2rVr2ykix5OdnY3RWPzHsMlkwmw22ykix1e3bl1CQkJYsGBB0b709HTWrFlD27Zt7RiZ4znbOdm9ezfz588nMDDQ3iGJlJr6T2Wn/pNtqP9Uduo/2Z76T7Zh776TpmU5mMGDBzNp0iR+/fVXvL29i+ZA+vr64u7ubufoHIO3t/cFc+w9PT0JDAzU3Psr8PTTT9OuXTvefPNNbr/9dtauXctnn33GZ599Zu/QHEafPn144403qFWrFo0aNeLPP/9k/PjxPPDAA/YOrVLLzMxkz549Rc8TExPZtGkTAQEB1KpVi2HDhvH6668TFRVF3bp1GT16NGFhYfTr189+QVdCl2rH0NBQbr31VjZu3MisWbMoLCws+n0TEBCAi4uLvcIWKRX1n8pO/SfbUP+p7NR/Kh31n8qu0vedKmRNLrEZ4F+3r7/+2t6hOTQt5Vk6v/32m6Vx48YWV1dXS3R0tOWzzz6zd0gOJT093TJ06FBLrVq1LG5ubpaIiAjLCy+8YMnNzbV3aJXaokWL/vXn4KBBgywWi3U5z9GjR1uCg4Mtrq6ulm7dull27txp36AroUu1Y2Ji4kV/3yxatMjeoYtcMfWfyof6T6Wj/lPZqP9UOuo/lV1l7zsZLBaLpXzSRiIiIiIiIiIiUt5Uc0dERERERERExIEpuSMiIiIiIiIi4sCU3BERERERERERcWBK7oiIiIiIiIiIODAld0REREREREREHJiSOyIiIiIiIiIiDkzJHRERERERERERB6bkjoiIiIiIiIiIA1NyR0QclsFgYMaMGfYOQ0RERMRhqP8kcnVSckdESuW+++7DYDBcsF133XX2Dk1ERESkUlL/SUTKi5O9AxARx3Xdddfx9ddfF9vn6upqp2hEREREKj/1n0SkPGjkjoiUmqurKyEhIcU2f39/wDrkd+LEifTu3Rt3d3ciIiKYNm1asfO3bt1K165dcXd3JzAwkEceeYTMzMxix3z11Vc0atQIV1dXQkNDGTJkSLHXjx8/Tv/+/fHw8CAqKoqZM2cWvXbq1CkGDhxI9erVcXd3Jyoq6oLOlIiIiEhFUv9JRMqDkjsiUm5Gjx7NLbfcwubNmxk4cCB33HEHO3bsACArK4tevXrh7+/PunXrmDp1KvPnzy/W+Zg4cSKDBw/mkUceYevWrcycOZN69eoVe49XX32V22+/nS1btnD99dczcOBATp48WfT+27dvZ86cOezYsYOJEydSrVq1imsAERERkSuk/pOIlIpFRKQUBg0aZDGZTBZPT89i2xtvvGGxWCwWwPLYY48VO6dNmzaWxx9/3GKxWCyfffaZxd/f35KZmVn0+uzZsy1Go9GSnJxssVgslrCwMMsLL7xw0RgAy4svvlj0PDMz0wJY5syZY7FYLJY+ffpY7r//ftvcsIiIiEgZqf8kIuVFNXdEpNSuvfZaJk6cWGxfQEBA0eO2bdsWe61t27Zs2rQJgB07dhAbG4unp2fR6+3bt8dsNrNz504MBgNHjhyhW7dul4yhadOmRY89PT3x8fEhNTUVgMcff5xbbrmFjRs30rNnT/r160e7du1Kda8iIiIitqD+k4iUByV3RKTUPD09Lxjmayvu7u4lOs7Z2bnYc4PBgNlsBqB3794cOHCA33//nfj4eLp168bgwYN55513bB6viIiISEmo/yQi5UE1d0Sk3KxevfqC5zExMQDExMSwefNmsrKyil5fsWIFRqORBg0a4O3tTZ06dViwYEGZYqhevTqDBg3i+++/Z8KECXz22Wdlup6IiIhIeVL/SURKQyN3RKTUcnNzSU5OLrbPycmpqOje1KlTiYuLo0OHDvzwww+sXbuWL7/8EoCBAwfy8ssvM2jQIF555RWOHTvGk08+yT333ENwcDAAr7zyCo899hhBQUH07t2bjIwMVqxYwZNPPlmi+F566SVatmxJo0aNyM3NZdasWUWdIxERERF7UP9JRMqDkjsiUmpz584lNDS02L4GDRqQkJAAWFdi+Omnn3jiiScIDQ3lxx9/pGHDhgB4eHgwb948hg4dSqtWrfDw8OCWW25h/PjxRdcaNGgQOTk5vPfee4wYMYJq1apx6623ljg+FxcXRo0axf79+3F3d6djx4789NNPNrhzERERkdJR/0lEyoPBYrFY7B2EiFx9DAYD06dPp1+/fvYORURERMQhqP8kIqWlmjsiIiIiIiIiIg5MyR0REREREREREQemaVkiIiIiIiIiIg5MI3dERERERERERByYkjsiIiIiIiIiIg5MyR0REREREREREQem5I6IiIiIiIiIiANTckdERERERERExIEpuSMiIiIiIiIi4sCU3BERERERERERcWBK7oiIiIiIiIiIOLD/B9n+GpZX3q8gAAAAAElFTkSuQmCC\n", | |
"text/plain": [ | |
"<Figure size 1400x500 with 2 Axes>" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
} | |
], | |
"source": [ | |
"# Plotting loss curves\n", | |
"\n", | |
"import matplotlib.pyplot as plt\n", | |
"from numpy import arange\n", | |
"\n", | |
"train_values = train_loss_dict.values()\n", | |
"val_values = val_loss_dict.values()\n", | |
"train_accuracy_values = train_accuracy_dict.values()\n", | |
"val_accuracy_values = val_accuracy_dict.values()\n", | |
"# Generate a sequence of integers to represent the epoch numbers\n", | |
"epochs = train_loss_dict.keys()\n", | |
"val_epochs = val_loss_dict.keys()\n", | |
"#plt.style.use('dark_background')\n", | |
"\n", | |
"fig, ax = plt.subplots(1, 2, figsize=(14, 5))\n", | |
"xd = np.array([])\n", | |
"yd = np.array([])\n", | |
"\n", | |
"ax[0].plot(epochs, train_values, label=\"Training Loss\")\n", | |
"ax[0].plot(val_epochs, val_values, label=\"Validation Loss\")\n", | |
"ax[0].set(xlabel='Epochs', ylabel='Loss')\n", | |
"ax[0].set_title('Loss Values')\n", | |
"ax[0].legend([\"Training\", \"Validation\"], loc=\"upper right\")\n", | |
"ax[0].grid()\n", | |
"\n", | |
"ax[1].plot(epochs, train_accuracy_values, label=\"Training Accuracy\")\n", | |
"ax[1].plot(val_epochs, val_accuracy_values, label=\"Validation Accuracy\")\n", | |
"ax[1].set(xlabel='Epochs', ylabel='Accuracy')\n", | |
"ax[1].set_title('Accuracy')\n", | |
"ax[1].legend([\"Training\", \"Validation\"], loc=\"lower right\")\n", | |
"ax[1].grid()\n", | |
"\n", | |
"plt.show()" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 13, | |
"id": "7fbf717b-2ee0-4030-882d-15dd851311d5", | |
"metadata": { | |
"executionInfo": { | |
"elapsed": 11564, | |
"status": "ok", | |
"timestamp": 1719242267847, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "7fbf717b-2ee0-4030-882d-15dd851311d5", | |
"jupyter": { | |
"source_hidden": true | |
}, | |
"tags": [] | |
}, | |
"outputs": [], | |
"source": [ | |
"# Translator\n", | |
"\n", | |
"from pickle import load\n", | |
"from tensorflow import Module\n", | |
"from keras.preprocessing.sequence import pad_sequences\n", | |
"from tensorflow import convert_to_tensor, int64, TensorArray, argmax, newaxis, transpose\n", | |
"\n", | |
"# Define the model parameters\n", | |
"h = 8 # Number of self-attention heads\n", | |
"d_k = 64 # Dimensionality of the linearly projected queries and keys\n", | |
"d_v = 64 # Dimensionality of the linearly projected values\n", | |
"d_model = 512 # Dimensionality of model layers' outputs\n", | |
"d_ff = 2048 # Dimensionality of the inner fully connected layer\n", | |
"n = 6 # Number of layers in the encoder stack\n", | |
"\n", | |
"def load_tokenizer(name):\n", | |
" with open(name, 'rb') as handle:\n", | |
" return load(handle)\n", | |
"\n", | |
"\n", | |
"# Define the dataset parameters\n", | |
"_, enc_seq_length, enc_vocab_size = load_tokenizer('tokenizer/enc_tokenizer.pkl')\n", | |
"_, dec_seq_length, dec_vocab_size = load_tokenizer('tokenizer/dec_tokenizer.pkl')\n", | |
"\n", | |
"#enc_seq_length = 7 # Encoder sequence length\n", | |
"#dec_seq_length = 12 # Decoder sequence length\n", | |
"#enc_vocab_size = 3288 # Encoder vocabulary size\n", | |
"#dec_vocab_size = 5275 # Decoder vocabulary size\n", | |
"\n", | |
"# Create model\n", | |
"inferencing_model = TransformerModel(enc_vocab_size, dec_vocab_size, enc_seq_length, dec_seq_length, h, d_k, d_v, d_model, d_ff, n, 0.1)\n", | |
"\n", | |
"class Translate(Module):\n", | |
" def __init__(self, model, **kwargs):\n", | |
" super(Translate, self).__init__(**kwargs)\n", | |
" self.transformer = model\n", | |
"\n", | |
" def __call__(self, sentence):\n", | |
" # Append start and end of string tokens to the input sentence\n", | |
" sentence[0] = \"<START> \" + sentence[0] + \" <EOS>\"\n", | |
"\n", | |
" # Load encoder and decoder tokenizers\n", | |
" enc_tokenizer, _, _ = load_tokenizer('tokenizer/enc_tokenizer.pkl')\n", | |
" dec_tokenizer, _, _ = load_tokenizer('tokenizer/dec_tokenizer.pkl')\n", | |
"\n", | |
" # Prepare the input sentence by tokenizing, padding and converting to tensor\n", | |
" encoder_input = enc_tokenizer.texts_to_sequences(sentence)\n", | |
" encoder_input = pad_sequences(encoder_input, maxlen=enc_seq_length, padding='post')\n", | |
" encoder_input = convert_to_tensor(encoder_input, dtype=int64)\n", | |
"\n", | |
" # Prepare the output <START> token by tokenizing, and converting to tensor\n", | |
" output_start = dec_tokenizer.texts_to_sequences([\"<START>\"])\n", | |
" output_start = convert_to_tensor(output_start[0], dtype=int64)\n", | |
"\n", | |
" # Prepare the output <EOS> token by tokenizing, and converting to tensor\n", | |
" output_end = dec_tokenizer.texts_to_sequences([\"<EOS>\"])\n", | |
" output_end = convert_to_tensor(output_end[0], dtype=int64)\n", | |
"\n", | |
" # Prepare the output array of dynamic size\n", | |
" decoder_output = TensorArray(dtype=int64, size=0, dynamic_size=True)\n", | |
" decoder_output = decoder_output.write(0, output_start)\n", | |
"\n", | |
" for i in range(dec_seq_length):\n", | |
"\n", | |
" # Predict an output token\n", | |
" prediction = self.transformer(encoder_input, transpose(decoder_output.stack()), training=False)\n", | |
"\n", | |
" prediction = prediction[:, -1, :]\n", | |
"\n", | |
" # Select the prediction with the highest score\n", | |
" predicted_id = argmax(prediction, axis=-1)\n", | |
" predicted_id = predicted_id[0][newaxis]\n", | |
"\n", | |
" # Write the selected prediction to the output array at the next available index\n", | |
" decoder_output = decoder_output.write(i + 1, predicted_id)\n", | |
"\n", | |
" # Break if an <EOS> token is predicted\n", | |
" if predicted_id == output_end:\n", | |
" break\n", | |
"\n", | |
" output = transpose(decoder_output.stack())[0]\n", | |
" output = output.numpy()\n", | |
"\n", | |
" output_str = []\n", | |
"\n", | |
" # Decode the predicted tokens into an output string\n", | |
" for i in range(output.shape[0]):\n", | |
"\n", | |
" key = output[i]\n", | |
" output_str.append(dec_tokenizer.index_word[key])\n", | |
"\n", | |
" if 'start' in output_str:\n", | |
" output_str.remove('start')\n", | |
" if 'eos' in output_str:\n", | |
" output_str.remove('eos')\n", | |
" return ' '.join(output_str)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 40, | |
"id": "8KVoRsgj4Drf", | |
"metadata": { | |
"colab": { | |
"base_uri": "https://localhost:8080/" | |
}, | |
"collapsed": true, | |
"executionInfo": { | |
"elapsed": 19483, | |
"status": "ok", | |
"timestamp": 1719244015082, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "8KVoRsgj4Drf", | |
"jupyter": { | |
"outputs_hidden": true, | |
"source_hidden": true | |
}, | |
"outputId": "4a359b79-4e55-41c4-8dfc-ab1aa05155e3" | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"English Input\t Model translation\t Actual translation\n", | |
"\n", | |
"let me see\t lass mich sehen\t lassen sie mich sehen\n", | |
"let me see\t lass mich sehen\t lasst mich sehen\n", | |
"let me see\t lass mich sehen\t lassen sie mich mal sehen\n", | |
"let me try\t fang es mir bitte\t lass mich versuchen\n", | |
"let me try\t fang es mir bitte\t lass mich mal probieren\n", | |
"let us out\t fang uns an\t\t lass uns heraus\n", | |
"let us out\t fang uns an\t\t lassen sie uns heraus\n", | |
"lets pray\t fang an uns zu zahlen\t lasset uns beten\n", | |
"lets talk\t fang uns an\t\t lass uns reden\n", | |
"look again\t fang noch wieder\t schau nochmal hin\n" | |
] | |
} | |
], | |
"source": [ | |
"# Inferencing the model\n", | |
"\n", | |
"from pickle import load\n", | |
"\n", | |
"with open('data/english-german.pkl', 'rb') as f:\n", | |
" data = load(f)\n", | |
"\n", | |
"#inferencing_model.load_weights('model_weights/epoch12.ckpt')\n", | |
"#translator = Translate(inferencing_model)\n", | |
"translator = Translate(training_model)\n", | |
"\n", | |
"prediction_matrix = [[pair[0], translator([pair[0]]), pair[1]] for pair in data[1000:1010]]\n", | |
"print(\"English Input\\t Model translation\\t Actual translation\")\n", | |
"print()\n", | |
"for i in prediction_matrix:\n", | |
" if len(i[1])<12:\n", | |
" print(f\"{i[0]}\\t {i[1]}\\t\\t {i[2]}\")\n", | |
" continue\n", | |
" print(f\"{i[0]}\\t {i[1]}\\t {i[2]}\")" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 33, | |
"id": "a7f71b47-ef52-48de-adb0-509d1e856e5c", | |
"metadata": { | |
"colab": { | |
"base_uri": "https://localhost:8080/" | |
}, | |
"collapsed": true, | |
"executionInfo": { | |
"elapsed": 2590, | |
"status": "ok", | |
"timestamp": 1719243604084, | |
"user": { | |
"displayName": "John Anchery", | |
"userId": "07897482772651248668" | |
}, | |
"user_tz": -330 | |
}, | |
"id": "a7f71b47-ef52-48de-adb0-509d1e856e5c", | |
"jupyter": { | |
"outputs_hidden": true, | |
"source_hidden": true | |
}, | |
"outputId": "c13cb1ba-6e6a-4d7e-f8d0-21cf2b1c3f53", | |
"tags": [] | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"fang ich bin arzt\n" | |
] | |
} | |
], | |
"source": [ | |
"# Try your own sentences\n", | |
"\n", | |
"sentence = ['im thirsty']\n", | |
"\n", | |
"#inferencing_model.load_weights('model_weights/epoch12.ckpt')\n", | |
"#translator = Translate(inferencing_model)\n", | |
"translator = Translate(training_model)\n", | |
"\n", | |
"print(translator(sentence))" | |
] | |
} | |
], | |
"metadata": { | |
"accelerator": "GPU", | |
"colab": { | |
"gpuType": "T4", | |
"provenance": [] | |
}, | |
"kernelspec": { | |
"display_name": "Python 3 (ipykernel)", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.12.4" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 5 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment