diff --git a/.gitignore b/.gitignore
index 5e7b83ec5..8efaa6142 100644
--- a/.gitignore
+++ b/.gitignore
@@ -175,3 +175,5 @@ sidebar.yml
token
token
*.pkg
+new.*
+old.*
diff --git a/README.md b/README.md
index c850fbf02..e9a96f4ec 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,6 @@
# tsai
+
diff --git a/nbs/015_data.mixed.ipynb b/nbs/015_data.mixed.ipynb
index 5e66cd015..a2e54bbfe 100644
--- a/nbs/015_data.mixed.ipynb
+++ b/nbs/015_data.mixed.ipynb
@@ -67,6 +67,8 @@
" if hasattr(dl, 'split_idxs'):\n",
" self.split_idxs = dl.split_idxs\n",
" dl.bs = self.bs\n",
+ " if i > 0 and hasattr(dl, 'get_idxs'):\n",
+ " dl.get_idxs = self.get_idxs_copy\n",
" dl.shuffle_fn = self.shuffle_fn\n",
" if self.c is None and hasattr(dl, \"c\"):\n",
" self.c = dl.c\n",
@@ -117,6 +119,9 @@
" outs += L(b[n_inp:])\n",
" self.y_idxs = self._get_vals(outs)\n",
"\n",
+ " def get_idxs_copy(self):\n",
+ " return self.loaders[0].get_idxs()\n",
+ "\n",
" def __iter__(self):\n",
" z = zip(*[_loaders[i.fake_l.num_workers == 0](i.fake_l)\n",
" for i in self.loaders])\n",
@@ -249,92 +254,92 @@
"
\n",
" 0 | \n",
" Private | \n",
- " Bachelors | \n",
- " Married-civ-spouse | \n",
- " 59.999999 | \n",
- " 131680.999115 | \n",
- " >=50k | \n",
+ " 5th-6th | \n",
+ " Separated | \n",
+ " 47.000000 | \n",
+ " 225065.000159 | \n",
+ " <50k | \n",
"
\n",
"
\n",
" 1 | \n",
" Private | \n",
- " 12th | \n",
- " Never-married | \n",
- " 18.000000 | \n",
- " 311795.000052 | \n",
+ " HS-grad | \n",
+ " Married-civ-spouse | \n",
+ " 56.999999 | \n",
+ " 84887.999356 | \n",
" <50k | \n",
"
\n",
"
\n",
" 2 | \n",
" Private | \n",
- " HS-grad | \n",
+ " Assoc-voc | \n",
" Married-civ-spouse | \n",
- " 45.000000 | \n",
- " 350440.002257 | \n",
+ " 30.000000 | \n",
+ " 176409.999275 | \n",
" >=50k | \n",
"
\n",
"
\n",
" 3 | \n",
- " Local-gov | \n",
- " Masters | \n",
- " Never-married | \n",
- " 44.000000 | \n",
- " 101593.001253 | \n",
+ " Private | \n",
+ " Some-college | \n",
+ " Married-civ-spouse | \n",
+ " 31.000000 | \n",
+ " 232474.999969 | \n",
" <50k | \n",
"
\n",
"
\n",
" 4 | \n",
- " ? | \n",
- " Some-college | \n",
- " Never-married | \n",
- " 20.999999 | \n",
- " 41355.995576 | \n",
+ " Private | \n",
+ " 10th | \n",
+ " Married-civ-spouse | \n",
+ " 26.000000 | \n",
+ " 293984.002897 | \n",
" <50k | \n",
"
\n",
"
\n",
" 5 | \n",
" Private | \n",
- " Bachelors | \n",
- " Never-married | \n",
- " 30.000000 | \n",
- " 207668.000292 | \n",
- " <50k | \n",
+ " HS-grad | \n",
+ " Married-civ-spouse | \n",
+ " 54.000000 | \n",
+ " 167770.000370 | \n",
+ " >=50k | \n",
"
\n",
"
\n",
" 6 | \n",
- " Federal-gov | \n",
+ " Private | \n",
" Bachelors | \n",
" Never-married | \n",
- " 28.000000 | \n",
- " 281859.998606 | \n",
+ " 25.000000 | \n",
+ " 60357.998190 | \n",
" <50k | \n",
"
\n",
"
\n",
" 7 | \n",
- " ? | \n",
- " Some-college | \n",
- " Never-married | \n",
- " 20.999999 | \n",
- " 180338.999810 | \n",
+ " Local-gov | \n",
+ " 7th-8th | \n",
+ " Married-civ-spouse | \n",
+ " 62.000000 | \n",
+ " 203524.999993 | \n",
" <50k | \n",
"
\n",
"
\n",
" 8 | \n",
" Private | \n",
" Some-college | \n",
- " Never-married | \n",
- " 20.000000 | \n",
- " 174713.999509 | \n",
+ " Married-civ-spouse | \n",
+ " 36.000000 | \n",
+ " 220510.999048 | \n",
" <50k | \n",
"
\n",
"
\n",
" 9 | \n",
- " Self-emp-not-inc | \n",
- " Bachelors | \n",
+ " State-gov | \n",
+ " Doctorate | \n",
" Married-civ-spouse | \n",
- " 50.000000 | \n",
- " 334273.005863 | \n",
- " <50k | \n",
+ " 55.000000 | \n",
+ " 120781.002923 | \n",
+ " >=50k | \n",
"
\n",
" \n",
""
@@ -362,7 +367,7 @@
"
\n",
" 0 | \n",
" False | \n",
- " 9.0 | \n",
+ " 10.0 | \n",
" <50k | \n",
"
\n",
"
\n",
@@ -375,30 +380,30 @@
" 2 | \n",
" False | \n",
" 13.0 | \n",
- " >=50k | \n",
+ " <50k | \n",
"
\n",
"
\n",
" 3 | \n",
" False | \n",
- " 9.0 | \n",
- " <50k | \n",
+ " 13.0 | \n",
+ " >=50k | \n",
"
\n",
"
\n",
" 4 | \n",
" False | \n",
- " 9.0 | \n",
+ " 10.0 | \n",
" <50k | \n",
"
\n",
"
\n",
" 5 | \n",
" False | \n",
- " 13.0 | \n",
- " >=50k | \n",
+ " 9.0 | \n",
+ " <50k | \n",
"
\n",
"
\n",
" 6 | \n",
" False | \n",
- " 10.0 | \n",
+ " 9.0 | \n",
" <50k | \n",
"
\n",
"
\n",
@@ -410,13 +415,13 @@
"
\n",
" 8 | \n",
" False | \n",
- " 13.0 | \n",
- " <50k | \n",
+ " 14.0 | \n",
+ " >=50k | \n",
"
\n",
"
\n",
" 9 | \n",
" False | \n",
- " 10.0 | \n",
+ " 9.0 | \n",
" <50k | \n",
"
\n",
" \n",
@@ -471,75 +476,75 @@
"
\n",
" \n",
" 0 | \n",
- " State-gov | \n",
- " HS-grad | \n",
- " Never-married | \n",
- " 43.000000 | \n",
- " 23156.998049 | \n",
- " <50k | \n",
+ " Private | \n",
+ " Masters | \n",
+ " Divorced | \n",
+ " 44.000000 | \n",
+ " 236746.000153 | \n",
+ " >=50k | \n",
"
\n",
" \n",
" 1 | \n",
" Private | \n",
- " 11th | \n",
- " Married-civ-spouse | \n",
- " 32.000000 | \n",
- " 140092.001434 | \n",
+ " Bachelors | \n",
+ " Never-married | \n",
+ " 22.000000 | \n",
+ " 189950.000000 | \n",
" <50k | \n",
"
\n",
" \n",
" 2 | \n",
- " Self-emp-not-inc | \n",
+ " Private | \n",
" HS-grad | \n",
- " Never-married | \n",
- " 43.000000 | \n",
- " 48086.995399 | \n",
+ " Married-civ-spouse | \n",
+ " 56.999999 | \n",
+ " 120302.001777 | \n",
" <50k | \n",
"
\n",
" \n",
" 3 | \n",
- " Self-emp-not-inc | \n",
- " Assoc-acdm | \n",
+ " Private | \n",
+ " HS-grad | \n",
" Never-married | \n",
- " 34.000000 | \n",
- " 177638.999728 | \n",
+ " 29.000000 | \n",
+ " 131087.999775 | \n",
" <50k | \n",
"
\n",
" \n",
" 4 | \n",
- " Local-gov | \n",
- " Masters | \n",
- " Married-civ-spouse | \n",
- " 65.000001 | \n",
- " 146453.999176 | \n",
+ " Self-emp-not-inc | \n",
+ " HS-grad | \n",
+ " Never-married | \n",
+ " 35.000000 | \n",
+ " 179171.000276 | \n",
" <50k | \n",
"
\n",
" \n",
" 5 | \n",
- " Private | \n",
+ " Self-emp-not-inc | \n",
" HS-grad | \n",
- " Married-civ-spouse | \n",
- " 33.000000 | \n",
- " 227281.999333 | \n",
+ " Divorced | \n",
+ " 75.000001 | \n",
+ " 242107.999406 | \n",
" <50k | \n",
"
\n",
" \n",
" 6 | \n",
" Private | \n",
- " HS-grad | \n",
+ " 12th | \n",
" Never-married | \n",
- " 33.000000 | \n",
- " 194900.999911 | \n",
+ " 36.000000 | \n",
+ " 137420.999182 | \n",
" <50k | \n",
"
\n",
" \n",
" 7 | \n",
" Private | \n",
- " HS-grad | \n",
- " Divorced | \n",
- " 23.000000 | \n",
- " 259301.002460 | \n",
- " <50k | \n",
+ " Doctorate | \n",
+ " Married-civ-spouse | \n",
+ " 35.000000 | \n",
+ " 189623.000011 | \n",
+ " >=50k | \n",
"
\n",
" \n",
""
@@ -573,7 +578,7 @@
"
\n",
" 1 | \n",
" False | \n",
- " 7.0 | \n",
+ " 9.0 | \n",
" <50k | \n",
"
\n",
"
\n",
@@ -585,18 +590,18 @@
"
\n",
" 3 | \n",
" False | \n",
- " 12.0 | \n",
+ " 9.0 | \n",
" <50k | \n",
"
\n",
"
\n",
" 4 | \n",
" False | \n",
- " 14.0 | \n",
+ " 9.0 | \n",
" <50k | \n",
"
\n",
"
\n",
" 5 | \n",
- " True | \n",
+ " False | \n",
" 10.0 | \n",
" <50k | \n",
"
\n",
@@ -609,8 +614,8 @@
"
\n",
" 7 | \n",
" False | \n",
- " 9.0 | \n",
- " <50k | \n",
+ " 10.0 | \n",
+ " >=50k | \n",
"
\n",
" \n",
""
@@ -633,82 +638,6 @@
"dls.train.show_batch()"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "((tensor([[ 8, 12, 5],\n",
- " [ 5, 2, 3],\n",
- " [ 7, 12, 5],\n",
- " [ 7, 8, 5],\n",
- " [ 3, 13, 3],\n",
- " [ 5, 12, 3],\n",
- " [ 5, 12, 5],\n",
- " [ 5, 12, 1]]),\n",
- " tensor([[ 0.3222, -1.5782],\n",
- " [-0.4850, -0.4696],\n",
- " [ 0.3222, -1.3418],\n",
- " [-0.3383, -0.1136],\n",
- " [ 1.9368, -0.4093],\n",
- " [-0.4117, 0.3570],\n",
- " [-0.4117, 0.0500],\n",
- " [-1.1455, 0.6606]])),\n",
- " (tensor([[1],\n",
- " [1],\n",
- " [1],\n",
- " [1],\n",
- " [1],\n",
- " [2],\n",
- " [1],\n",
- " [1]]),\n",
- " tensor([[-0.4258],\n",
- " [-1.2097],\n",
- " [-0.4258],\n",
- " [ 0.7502],\n",
- " [ 1.5342],\n",
- " [-0.0338],\n",
- " [-0.4258],\n",
- " [-0.4258]])))"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "xb, yb = first(dls.train)\n",
- "xb"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "(torch.Size([8, 3]),\n",
- " torch.Size([8, 2]),\n",
- " torch.Size([8, 1]),\n",
- " torch.Size([8, 1]))"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "xs, ys = first(dls.train)\n",
- "xs[0][0].shape, xs[0][1].shape, xs[1][0].shape, xs[1][1].shape"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -725,27 +654,44 @@
"metadata": {},
"outputs": [
{
- "data": {
- "text/plain": [
- "(TSTensor(samples:8, vars:2, len:5, device=cpu, dtype=torch.float32),\n",
- " tensor([7., 0., 2., 1., 5., 4., 3., 6.]))"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor([[[5., 5., 5., 5., 5.],\n",
+ " [5., 5., 5., 5., 5.]],\n",
+ "\n",
+ " [[0., 0., 0., 0., 0.],\n",
+ " [0., 0., 0., 0., 0.]],\n",
+ "\n",
+ " [[4., 4., 4., 4., 4.],\n",
+ " [4., 4., 4., 4., 4.]],\n",
+ "\n",
+ " [[3., 3., 3., 3., 3.],\n",
+ " [3., 3., 3., 3., 3.]]], device='mps:0') tensor([5., 0., 4., 3.], device='mps:0')\n",
+ "tensor([[[6., 6., 6., 6., 6.],\n",
+ " [6., 6., 6., 6., 6.]],\n",
+ "\n",
+ " [[1., 1., 1., 1., 1.],\n",
+ " [1., 1., 1., 1., 1.]],\n",
+ "\n",
+ " [[2., 2., 2., 2., 2.],\n",
+ " [2., 2., 2., 2., 2.]],\n",
+ "\n",
+ " [[7., 7., 7., 7., 7.],\n",
+ " [7., 7., 7., 7., 7.]]], device='mps:0') tensor([6., 1., 2., 7.], device='mps:0')\n"
+ ]
}
],
"source": [
- "X = np.repeat(np.repeat(np.arange(8)[:, None, None], 2, 1), 5, 2).astype(float)\n",
- "X = np.concatenate([X, X])\n",
+ "X = np.repeat(np.repeat(np.arange(16)[:, None, None], 2, 1), 5, 2).astype(float)\n",
"y = np.concatenate([np.arange(len(X)//2)]*2)\n",
"alphabet = np.array(list(string.ascii_lowercase))\n",
"# y = alphabet[y]\n",
"splits = TimeSplitter(.5, show_plot=False)(range_of(X))\n",
"tfms = [None, TSRegression()]\n",
- "dls1 = get_ts_dls(X, y, splits=splits, tfms=tfms)\n",
- "dls1.one_batch()"
+ "dls1 = get_ts_dls(X, y, splits=splits, tfms=tfms, bs=4)\n",
+ "for xb, yb in iter(dls1.train):\n",
+ " print(xb.data, yb)"
]
},
{
@@ -755,69 +701,211 @@
"outputs": [
{
"data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " cat1 | \n",
+ " cat2 | \n",
+ " cont | \n",
+ " target | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 0.0 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 10 | \n",
+ " 100.0 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " 2 | \n",
+ " 20 | \n",
+ " 200.0 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " 3 | \n",
+ " 30 | \n",
+ " 300.0 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 4 | \n",
+ " 40 | \n",
+ " 400.0 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ " 5 | \n",
+ " 5 | \n",
+ " 50 | \n",
+ " 500.0 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ " 6 | \n",
+ " 6 | \n",
+ " 60 | \n",
+ " 600.0 | \n",
+ " 6 | \n",
+ "
\n",
+ " \n",
+ " 7 | \n",
+ " 7 | \n",
+ " 70 | \n",
+ " 700.0 | \n",
+ " 7 | \n",
+ "
\n",
+ " \n",
+ " 8 | \n",
+ " 8 | \n",
+ " 80 | \n",
+ " 800.0 | \n",
+ " 0 | \n",
+ "
\n",
+ " \n",
+ " 9 | \n",
+ " 9 | \n",
+ " 90 | \n",
+ " 900.0 | \n",
+ " 1 | \n",
+ "
\n",
+ " \n",
+ " 10 | \n",
+ " 10 | \n",
+ " 100 | \n",
+ " 1000.0 | \n",
+ " 2 | \n",
+ "
\n",
+ " \n",
+ " 11 | \n",
+ " 11 | \n",
+ " 110 | \n",
+ " 1100.0 | \n",
+ " 3 | \n",
+ "
\n",
+ " \n",
+ " 12 | \n",
+ " 12 | \n",
+ " 120 | \n",
+ " 1200.0 | \n",
+ " 4 | \n",
+ "
\n",
+ " \n",
+ " 13 | \n",
+ " 13 | \n",
+ " 130 | \n",
+ " 1300.0 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ " 14 | \n",
+ " 14 | \n",
+ " 140 | \n",
+ " 1400.0 | \n",
+ " 6 | \n",
+ "
\n",
+ " \n",
+ " 15 | \n",
+ " 15 | \n",
+ " 150 | \n",
+ " 1500.0 | \n",
+ " 7 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
"text/plain": [
- "(tensor([[2, 2],\n",
- " [5, 5],\n",
- " [1, 1],\n",
- " [7, 7],\n",
- " [3, 3],\n",
- " [6, 6],\n",
- " [8, 8],\n",
- " [4, 4]]),\n",
- " tensor([[100.],\n",
- " [400.],\n",
- " [ 0.],\n",
- " [600.],\n",
- " [200.],\n",
- " [500.],\n",
- " [700.],\n",
- " [300.]]),\n",
- " tensor([[1],\n",
- " [4],\n",
- " [0],\n",
- " [6],\n",
- " [2],\n",
- " [5],\n",
- " [7],\n",
- " [3]], dtype=torch.int8))"
+ " cat1 cat2 cont target\n",
+ "0 0 0 0.0 0\n",
+ "1 1 10 100.0 1\n",
+ "2 2 20 200.0 2\n",
+ "3 3 30 300.0 3\n",
+ "4 4 40 400.0 4\n",
+ "5 5 50 500.0 5\n",
+ "6 6 60 600.0 6\n",
+ "7 7 70 700.0 7\n",
+ "8 8 80 800.0 0\n",
+ "9 9 90 900.0 1\n",
+ "10 10 100 1000.0 2\n",
+ "11 11 110 1100.0 3\n",
+ "12 12 120 1200.0 4\n",
+ "13 13 130 1300.0 5\n",
+ "14 14 140 1400.0 6\n",
+ "15 15 150 1500.0 7"
]
},
- "execution_count": null,
"metadata": {},
- "output_type": "execute_result"
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor([[5, 5],\n",
+ " [5, 5],\n",
+ " [6, 6],\n",
+ " [4, 4]], device='mps:0') tensor([[400.],\n",
+ " [400.],\n",
+ " [500.],\n",
+ " [300.]], device='mps:0') tensor([[4],\n",
+ " [4],\n",
+ " [5],\n",
+ " [3]], device='mps:0', dtype=torch.int8)\n",
+ "tensor([[4, 4],\n",
+ " [7, 7],\n",
+ " [2, 2],\n",
+ " [1, 1]], device='mps:0') tensor([[300.],\n",
+ " [600.],\n",
+ " [100.],\n",
+ " [ 0.]], device='mps:0') tensor([[3],\n",
+ " [6],\n",
+ " [1],\n",
+ " [0]], device='mps:0', dtype=torch.int8)\n"
+ ]
}
],
"source": [
- "data = np.concatenate([np.repeat(np.arange(8)[:, None], 3, 1)*np.array([1, 10, 100])]*2)\n",
+ "data = np.repeat(np.arange(16)[:, None], 3, 1)*np.array([1, 10, 100])\n",
"df = pd.DataFrame(data, columns=['cat1', 'cat2', 'cont'])\n",
"df['cont'] = df['cont'].astype(float)\n",
"df['target'] = y\n",
+ "display(df)\n",
"cat_names = ['cat1', 'cat2']\n",
"cont_names = ['cont']\n",
"target = 'target'\n",
"dls2 = get_tabular_dls(df, procs=[Categorify, FillMissing, #Normalize\n",
- " ], cat_names=cat_names, cont_names=cont_names, y_names=target, splits=splits, bs=8)\n",
- "dls2.one_batch()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "((TSTensor(samples:8, vars:2, len:5, device=cpu, dtype=torch.float32), tensor([7., 0., 2., 1., 5., 4., 3., 6.])),)\n"
- ]
- }
- ],
- "source": [
- "z = zip(_loaders[dls1.train.fake_l.num_workers == 0](dls1.train.fake_l))\n",
- "for b in z: \n",
- " print(b)\n",
- " break"
+ " ], cat_names=cat_names, cont_names=cont_names, y_names=target, splits=splits, bs=4)\n",
+ "for b in iter(dls2.train):\n",
+ " print(b[0], b[1], b[2])"
]
},
{
@@ -843,6 +931,59 @@
"test_eq(tensor(y[dl.input_idxs]), yb.long().cpu())"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "tensor([[[0., 0., 0., 0., 0.],\n",
+ " [0., 0., 0., 0., 0.]],\n",
+ "\n",
+ " [[1., 1., 1., 1., 1.],\n",
+ " [1., 1., 1., 1., 1.]],\n",
+ "\n",
+ " [[2., 2., 2., 2., 2.],\n",
+ " [2., 2., 2., 2., 2.]],\n",
+ "\n",
+ " [[4., 4., 4., 4., 4.],\n",
+ " [4., 4., 4., 4., 4.]]], device='mps:0') (tensor([[1, 1],\n",
+ " [2, 2],\n",
+ " [3, 3],\n",
+ " [5, 5]], device='mps:0'), tensor([[ 0.],\n",
+ " [100.],\n",
+ " [200.],\n",
+ " [400.]], device='mps:0')) tensor([0., 1., 2., 4.], device='mps:0')\n",
+ "tensor([[[3., 3., 3., 3., 3.],\n",
+ " [3., 3., 3., 3., 3.]],\n",
+ "\n",
+ " [[5., 5., 5., 5., 5.],\n",
+ " [5., 5., 5., 5., 5.]],\n",
+ "\n",
+ " [[6., 6., 6., 6., 6.],\n",
+ " [6., 6., 6., 6., 6.]],\n",
+ "\n",
+ " [[7., 7., 7., 7., 7.],\n",
+ " [7., 7., 7., 7., 7.]]], device='mps:0') (tensor([[4, 4],\n",
+ " [6, 6],\n",
+ " [7, 7],\n",
+ " [8, 8]], device='mps:0'), tensor([[300.],\n",
+ " [500.],\n",
+ " [600.],\n",
+ " [700.]], device='mps:0')) tensor([3., 5., 6., 7.], device='mps:0')\n"
+ ]
+ }
+ ],
+ "source": [
+ "bs = 4\n",
+ "dls = get_mixed_dls(dls1, dls2, bs=bs)\n",
+ "for xb, yb in iter(dls.train):\n",
+ " print(xb[0].data, xb[1], yb)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -862,9 +1003,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "/Users/nacho/notebooks/tsai/nbs/022_data.mixed.ipynb saved at 2022-11-09 12:51:54\n",
+ "/Users/nacho/notebooks/tsai/nbs/015_data.mixed.ipynb saved at 2025-01-20 09:51:26\n",
"Correct notebook to script conversion! 😃\n",
- "Wednesday 09/11/22 12:51:57 CET\n"
+ "Monday 20/01/25 09:51:29 CET\n"
]
},
{
diff --git a/nbs/021_calibration.ipynb b/nbs/021_calibration.ipynb
index 57cae0244..5b1bb66ef 100644
--- a/nbs/021_calibration.ipynb
+++ b/nbs/021_calibration.ipynb
@@ -65,7 +65,7 @@
" def __init__(self, model, lr=0.01, max_iter=1_000, line_search_fn=None, n_bins=10, verbose=True):\n",
" super().__init__()\n",
" self.model = ModelWithTemperature(model) if not hasattr(model, 'temperature_scale') else model\n",
- " self.lr, self.max_iter, self.line_search_fn, self.n_bins, self.verbose = lr, max_iter, line_search_fn, n_bins, verbose \n",
+ " self.lr, self.max_iter, self.line_search_fn, self.n_bins, self.verbose = lr, max_iter, line_search_fn, n_bins, verbose\n",
" self.nll_criterion = CrossEntropyLossFlat()\n",
" self.ece_criterion = ECELoss(n_bins)\n",
"\n",
@@ -181,11 +181,11 @@
"source": [
"#|export\n",
"@patch\n",
- "def calibrate_model(self:Learner, X=None, y=None, lr=1e-2, max_iter=10_000, line_search_fn=None, n_bins=10, strategy='uniform', \n",
+ "def calibrate_model(self:Learner, X=None, y=None, lr=1e-2, max_iter=10_000, line_search_fn=None, n_bins=10, strategy='uniform',\n",
" show_plot=True, figsize=(6,6), verbose=True):\n",
- " if X is not None and y is not None: \n",
+ " if X is not None and y is not None:\n",
" dl = self.dls.valid.new_dl(X, y)\n",
- " else: \n",
+ " else:\n",
" dl = self.dls.valid\n",
" assert dl.c == 2, \"calibrate_model is only available for binary classification tasks\"\n",
" temp_setter = TemperatureSetter(self.model, lr=lr, max_iter=max_iter, line_search_fn=line_search_fn, n_bins=n_bins, verbose=verbose)\n",
@@ -209,6 +209,33 @@
"execution_count": null,
"metadata": {},
"outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n"
+ ],
+ "text/plain": [
+ "
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
{
"data": {
"text/html": [
@@ -225,17 +252,17 @@
" \n",
" \n",
" 0 | \n",
- " 0.696826 | \n",
- " 0.706016 | \n",
- " 0.430000 | \n",
- " 00:04 | \n",
+ " 0.724956 | \n",
+ " nan | \n",
+ " nan | \n",
+ " 00:00 | \n",
"
\n",
" \n",
" 1 | \n",
- " 0.690209 | \n",
- " 0.699720 | \n",
- " 0.490000 | \n",
- " 00:03 | \n",
+ " 0.713688 | \n",
+ " nan | \n",
+ " nan | \n",
+ " 00:00 | \n",
"
\n",
" \n",
""
@@ -246,6 +273,14 @@
},
"metadata": {},
"output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/nacho/notebooks/tsai/tsai/data/core.py:648: RuntimeWarning: overflow encountered in scalar add\n",
+ " b = slice(b[0], min(self.n, b[0] + self.bs))\n"
+ ]
}
],
"source": [
@@ -266,24 +301,22 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Before temperature - NLL: 0.700, ECE: 0.066\n",
+ "Before temperature - NLL: 0.696, ECE: 0.032\n",
"Calibrating the model...\n",
"...model calibrated\n",
- "Optimal temperature: 6.383\n",
- "After temperature - NLL: 0.693, ECE: 0.019\n",
+ "Optimal temperature: 3.641\n",
+ "After temperature - NLL: 0.693, ECE: 0.018\n",
"\n"
]
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAGhCAYAAACdyeicAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABZ8ElEQVR4nO3dd5xU1fn48c/DAlJ2YWkiHQSUXmTFggKCBSxU6WU3JuFnjGl+Y8w3JsaY+E2+qd8UE2MSs2yjC4JgR0QiCqggAiJt6b0sZSlbnt8f9y4Ow5a77Mzc2Z3n/XrNi5l7zz33mTvLPHPPufccUVWMMcYYL6r5HYAxxpjKw5KGMcYYzyxpGGOM8cyShjHGGM8saRhjjPHMkoYxxhjPLGmYsBCRW0RktojsE5ELInJURN4UkWQRiStnXU+LiAYtUxF5OriMiFQP0VsoLZ4UEXmohOUqIm3DHcOVKO44etyul7ttw3DEZSoXSxom5ETku8B/gIbAE8CdwEPAF8DfgPtDsJtbgH+GoJ4rkYLzfoItxolrf0SjCb9ewE9xPk8T48L+q8zEFhHpD/we+Iuqfjto9csi8nugbkX3o6ofVLSOIiJylaqer2g9qnoYOByCkIyJWnamYULtCeAY8IPiVqrqNlX9FEBEmojI30XkCxHJFZHdIpIlIi3K2klw81SAziLyjlvffhF5RkSqBWw30N12lIj8Q0QOAwfddR1EJF1EdojIWRHZLiJ/E5EGAdsvAwYA/dx61F1WbPOUiNQQkV+ISLbbTJftvq4RUKatu93/c+PdLyInRGSRiLT0cCyWicgKERkuIp+JyHkR+VxExnrYtp6I/MVtRjwvIptF5HsiIkXvCfi3W3xLwHtuW1KdpmqzMw0TMm5fxR3AAlU952GThsA54L9xfqE3B/4L+I+IdPJYR7AFwIvAL4F7gJ8AhcDTQeX+DLwKTAFqucuaA7uB7wLHgWuBHwFLcJqdAB4BMoA44P+5y06WEs90YCzwP8AK4FbgSbfuiUFl/xt4H6fp62rgd+6+Bpb2hl0dgD/hvM9DwDeAmSJyWFXfKW4DN5kuBm4AngLWA/fhnCk2wXnvi4FfAD8GxgB73M2rWhOc8UpV7WGPkDyApoACv7zC7eOAVm4dIwOWP+38qV5SVoGng8sAPwwq9w/gFJDovh7olpvvIZ7qwG1u+d4By5cBK4opn+KWbeu+7hYcp7v8x+7yHu7rtu7rZUHlvu8ub15GnMvccjcHHcvPgfdKOo44fUsKpATV90/gPNA46H118PtvzB7+P6x5yvhKRL4hIutE5DSQD+xyV11/hVXODno9E4jH+QIPNL+YWGqKyI/cpp2zQB7wXgXi6e/+mxG0vOj1gKDlS4Jer3f/be1hX7s1oJ9HVQuAOUDfwOa5YuIrBLKKia8mX55dGXORJQ0TSkeBs0AbL4VF5FvAX4G3gFFAX+Bmd3WtkrYrw8ESXgf3kxTXvPJLnF/jGTjNNH3duK40nqKrjYL3dSBofZFjQa+LOue97Dv4fRctq4nT1FRSfMdU9YLH+IyxPg0TOqqa73YK3+XxiqTxwNuq+l9FC0SkXQXDaApsD3oNsDc43BLiSVPVXwTEE1+BWIqSwDXAtoDl1wStD4WmJSy7QMlXdB0DGopIzaDEEY74TBVhZxom1H4FNAJ+XdxKEWknIj3cl3VwmoACfaWC+w++Ymg8cJovm3pK4zWe80BtD/UtD4gh0CT332Ue6vCqlYgUnaUVXZQwBlilqoUlbPMuznfAmGLiuwCsdF8XJX8v79lUcXamYUJKVZeLyGPA70WkC5CK00/RABgMfA3nqqFPgdeAJ0TkR8AqYBDwYAVD+Lrbhr8a5+qpr+F0ROd42PY1IFlE1gNbcZqmbi2m3EbgEREZh3MGcUpVNwcXUtXPRGQG8LR7p/r7OP0EPwFmqKqXRObVQWCWiPwU58ziG8B17r8leRXniq7nRaQJsAG4F+eY/VJVj7jlNrr/flNEpuMk1k+LadYyMcCShgk5Vf0/EVkFfA/4LdAY5wqmNTiXqS5yiz4DJLrlauH88r2HS5uXyms4zuW0PwFycC4X/bnHbb8FCPCs+3oJMAEnoQX6X5yO8X/idLK/S8mXxabgvJ+HcK6a2udu/zOPMXm1Fefs7n+AjkA2MEFLuNwWQFULReQ+d5sncM4Qs4HHgP8LKLfOvSdmGvB1nLOTdm5ZE2NE1aZ7NaYyc/uRqqvqbX7HYqo+69MwxhjjmSUNY4wxnlnzlDHGGM/sTMMYY4xnljSMMcZ4ZknDGGOMZ5Y0jDHGeGZJwxhjjGeWNIwxxnhmScMYY4xnljSMMcZ4ZknDGGOMZ5Y0jDHGeGZJwxhjjGeWNCoxEdkgIgNDUVZEXhWRZI91ZYvInV7KGm9E5BsiclBETotII/ffa0NQ79MikhGKGI0BSxqVmqp2VdVl5S1b3BeJqg5V1ekVjUlEUkXkgvulV/RYF7C+prv/LSJyxk1AL4pIW3f9MhE5F7T9ohJ3ePn+J4rITrfuBSLSsJSyvUTkIxHJdf/tFbT+BhFZ7sZwUES+E7T+OyKyw93XJhG5zmucQfXUAH4P3K2q8ap61P23IpNRednvQBEpDDrWp0XkloAy97jH4JSIHBaRd0VkmLsuRUQKitm+ucf9txWRd9zj/3lpP0RE5Cr37+SkiBxwZ4csWldTROa6f0sa/ONIRBJFZLqIHHIfT5fzUJkAljRMOPza/dIrevQMWDcXGIYz5Wt9oCfwEc5UsEUeDdr+AS87FZGuwN+BKUBTIBf4awllawIvAxk4U9FOB152lyMijXGmf/07zox2HYA3Arb/GvBV4D6c2fvuB45wZZrizFy44Qq3r4h9Qcc6XlVXAojIg8AcIA1o6cb5FBD4eawsZvt9Hvc9A/gE5/g+Ccx1p50tztM4MxK2Ae4AfiAiQwLWrwAmAweK2fYPOPO/twX6AlNEpKJz0ccuVbVHJX3gTLd5p/v8aWA2zn/wUzhfQEnBZYEhwAWceZ5PA+vc9cuAr7nP2wNLgaM4X4SZQGJx+y0mplTgFyWsuxM4C7Qq5T1djOMKjsf/AFkBr9u77zWhmLJ3A3txpwdwl+0ChgTUlV7CfqoBu4HBIfgMrwPOAOp+Hkvd5Qp0CDimzwGL3c/2Q6B9QB1/dOM5iZOAbw9Y9zSQUcK+BwJ7Slgn7vF4vJTYU4AVFXjf5wM/G+A94OESyu/DORMrev1zYGYx5fYAA4OWHQFuDHj9I+C9in52sfqwM42qZRgwE2fe7YXAX4ILqOprOF+Is/Tys4AiAvwSaA50BlrhfPlU1J3AKlXdfaUViMgJESlpWtOuwMWmMFXdhpM0ims26gp8qu63iOtTdznAzcAxEXnfbdJYJCKt3XUt3Uc3EdntNlH9TETK/f9JVb8I2Geiqg4qoeh4nHnFG+DMB/5swLrVQC+gIZAFzBGRWuWNJcj1OJ/73CutQET+KiLFnunhvOftqnoqYNk6vjwWgfU0AJoR8NmWVLa0cIKedyvHtiaAJY2qZYWqLlHVAiAdp+mn3FR1q6q+qarnVfUwTnv7gHJU8X33y73oUdRX0gjY72H7PwVt//OA2BJVdUUJ28UDOUHLcoCEKyjbEkgGvgO0BnbgNKcUrQPnbKU7TnPJBJzmqnCZr6qrVDUf58yvV9EKVc1Qpx8kX1V/B1yF86XvRfOgY31CROrifFZQ9ud1c9C22wLiekRVHylhu/J+VkXryypbnNeAH4pIgoh0AB7Caa4yV8CSRtUS2J6bC9QSkerlrUREmorITBHZKyIncdr9G5ejit+6X+5Fj6Krso7i/GIsy7eDtv+Jx/2eBuoFLauH06RT3rJncb6oV6vqOZxf+beKSH13HTh9NydUNRun7+Pe4oIK6iRuXVwZD4I/26IvUkTk+25HfI6InMDpK/L6ee0LOtaJqnoG57OCsj+vD4K2be9xv+X9rIrWl1W2ON/G+cy24PRjzcBpxjJXwJJGbCprjt//cct0V9V6OB2MUvomnrwF9BWRlmWWvDIbCDi7EueS1auAL0oo20NEAt9XD77sjP6US49T4PPNOM1eJa2/hF7aSbzLyxvxSkRuB34AjAUaqGoizq/win5em3H6SUZXsJ6SbACuFZHAs4WeFHMxgKoexznj6VlW2eKo6jFVnaSq16hqV5zvvVVXHHmMs6QRmw4CbUtpg0/A+XWXIyItgMdDsVNVfQt4E5gvIn1EpLrbZPCwiDwUgl1kAg+IyO1uE8szwEtB7eZFlgEFwLfdyzkfdZcvdf/9NzBSnMtyawA/wWn+y1HVXGAWzhU8CW4SnAa8EoL3UF4JQD5wGKguIk9x+S/4cnP7eh4DfiIiXxGReiJSTURuE5EXQlD/F8Ba4KciUktERuIk7XklbJIG/FhEGohIJ+DrOBcIABcvyS3qx6np1inuuvbi3PsSJyJDcT6rX1T0PcQqSxqxaY7771ER+biY9T8DbsD5xboYeKmc9f8gqEkm8FLUB4ElOF+6OcBnQBLOWUiRvwRt/1HRCvf17cXtVFU3AA/jJI9DOF+ojwRs+6qI/MgtewEYAUwFTuC0c49wl6OqS3Guslns1tUB5zLhIo/iJNZ9wEqcDugXPR+h0Hkdp83+C2AncA7nDMGr5nL5fRajAVR1LjAO59jsw/mx8QucJp4itxSz/Y0AIvK8iDxfyr7H43z2x4FfAQ+6fWiIyCQRCTyT+CmwzX2P7wK/cS/qKLIZpwmqhXtMzuJcngvQB1iP05z1S2CS+7diroBcevGIMcYYUzI70zDGGONZRJKGOLf/HxKRz0pYLyLyJxHZKiKfisgNkYjLGGNM+UTqTCMV507kkgzFGSKgI04n1d8iEJMxxphyikjSUNXlwLFSigwH0tTxAZAoIl6u5zfGGBNB0dKn0YJLr/jY4y4zxhgTRcp9t7DfRGQaThMWdevW7dOpUyefIzJR4dhHJa9r2CdycRgTpVSV48ePIyJs3779iKqWNKJwqaIlaezFGRytSEt32WVU9QXgBYCkpCRds2ZN+KMz0W9BW8jdefnyOm1ghP2NmNiWl5fHrFmzqFWrFiNHjqR69erF/GfxJlqapxYCU92rqG4GclTVy8B2xjh6PgtxQWPQxdVxlhsTwy5cuEBWVhZ16tRh1KhRxMXFVai+iJxpiMgMnLH7G4vIHpy7O2sAqOrzOHcI34sz5HMuYBOkmPJpN8n5d92TzhlHrWbQ+zdfLjcmBp0/f56srCwaNmzIAw88QLVqFT9PqNR3hFvzlCnWfyZBs7vg2hS/IzHGN+fOnSMzM5OmTZty3333ETg2p4h8pKpJV1JvtPRphExeXh579uzh3LlzfodiwqhWrVq0bNmSGjVqXL6yQU84/mnkgzImSpw9e5aMjAxatmzJkCFDLkkYFVXlksaePXtISEigbdu2IT1QJnqoKkePHmXPnj20a9fu8gKJPWD/G5cvNyYG5Obmkp6eTrt27bjrrrtC/j0YLR3hIXPu3DkaNWpkCaMKExEaNWpU8tlkYg84sQ4qcdOrMVfi9OnTpKam0qFDh7AkDKiCSQOwhBEDSv2Ma7uDCZw7UHIZY6qYU6dOMX36dLp27cqgQYPC9j1YJZNGZRcfH192IVMyEeds4/g6vyMxJiJycnJITU2lZ8+eDBgwIKw/nC1p7Mh0bgzLqub8uyMzIrstKCiIyH5iVmJPOGGd4abqO3HiBKmpqSQlJXHbbbeFfX+xnTR2ZMKqae6dxOr8u2pahRNHdnY2nTp1YtKkSXTu3JkHH3yQ3Nxc2rZtyxNPPMENN9zAnDlzmDFjBt27d6dbt2488cQTl9Txve99j65duzJ48GAOHz4MwD/+8Q9uvPFGevbsyejRo8nNzQVgzpw5dOvWjZ49e9K/f/8KxV5lNOhhScNUeceOHSM1NZVbbrmFW265JSL7jO2kse5JKMi9dFlBrrO8gjZv3swjjzzCpk2bqFevHn/9618BaNSoER9//DH9+/fniSeeYOnSpaxdu5bVq1ezYMECAM6cOUNSUhIbNmxgwIAB/OxnPwNg1KhRrF69mnXr1tG5c2f+9a9/AfDMM8/w+uuvs27dOhYuXFjh2KsEa54yVdyRI0eYPn06t99+O3379o3YfqvcJbeXybqCtr3cnaVvN7Hsq3JatWpFv379AJg8eTJ/+tOfABg3bhwAq1evZuDAgTRp4owZNmnSJJYvX86IESOoVq3axXKTJ09m1KhRAHz22Wf8+Mc/5sSJE5w+fZp77rkHgH79+pGSksLYsWMvlo159bvA6a1QcB7irvI7GmNC6tChQ2RkZDBo0CB69eoV0X1X/aRR2hd8qYPcZVdot8EdUUWv69ate8V1paSksGDBAnr27ElqairLli0D4Pnnn+fDDz9k8eLF9OnTh48++ohGjRpVKP5KL64WxF8LJzdBg15+R2NMyBw4cIDMzEzuvvtuunfvHvH9x3bzVBgHudu1axcrV64EICsr67IOqr59+/Luu+9y5MgRCgoKmDFjBgMGDACgsLCQuXPnXrbtqVOnaNasGXl5eWRmftnvsm3bNm666SaeeeYZmjRpwu7duzE4neF2Z7ipQvbt20dGRgZDhw71JWFArCeNdpOg7wvOmQXi/Nv3hZAMcnf99dfz3HPP0blzZ44fP843vvGNS9Y3a9aMX/3qV9xxxx307NmTPn36MHz4cMA5G1m1ahXdunVj6dKlPPXUUwD8/Oc/56abbqJfv34EziPy+OOPX+xQv/XWW+nZs2eF468SEq0z3FQde/bsISsri/vvv58uXbr4FkeVG7Bw06ZNdO7c2aeIHNnZ2dx///189tlnvsZR1ZX5We9dApv/AIPejFxQxoTBrl27mDVrFiNGjKBjx44Vrs8GLDSmOA3sXg1T+WVnZzNnzhxGjRpF+/bt/Q4nxpunwqRt27Z2lhENajeHwnw4a8OJmMpp27ZtzJkzhzFjxkRFwgBLGqYqE7GzDVNpbdmyhZdeeolx48bRtm1bv8O5yJKGqdqsM9xUQp9//jkvv/wyEyZMoHXr1n6HcwlLGqZqszvDTSWzYcMGXnnlFSZOnEjLli39DucyljRM1WbNU6YSWb9+Pa+99hqTJ0+mefPmfodTLEsaPktNTeXRRx8FnDu709LSABg4cCAVmf88OzubrKyscm+XkpJy8cbCUApXvWWq1wVOfQEFFyK/b2PKYe3atbz55ptMmTKFa665xu9wShTzl9yuz1zP20++Tc6uHOq3rs/gZwfTfZI/d1o+/PDD5Sqfn59P9erFf4RFSWPixImhCK3C8fimem2o2w5Ofu6MfGtMFProo49Yvnw5U6dOpXHjxn6HU6qYPtNYn7meRdMWkbMzBxRyduawaNoi1meur1C9aWlp9OjRg549ezJlyhQAFi1axE033UTv3r258847OXjw4GXbPf300/z2t7+9+Do9PZ1evXrRrVs3Vq1adbHMlClT6NevH1OmTCE7O5vbb7+dG264gRtuuIH3338fgB/+8Ie899579OrViz/84Q8UFBTw+OOPc+ONN9KjRw/+/ve/A858248++ijXX389d955J4cOHSr2PQ0cOJDvfOc7nuIZNGgQPXr0YPDgwezatetiHW+99RZJSUlcd911vPLKKwAlxr9//3769+9/cX/vvffelX8g1hluotiqVat47733SE5OjvqEATF+pvH2k2+Tl5t3ybK83DzefvLtKz7b2LBhA7/4xS94//33ady4MceOHQPgtttu44MPPkBE+Oc//8mvf/1rfve735VaV25uLmvXrmX58uU89NBDF+/92LhxIytWrKB27drk5uby5ptvUqtWLbZs2cKECRNYs2YNv/rVr/jtb3978cv5hRdeoH79+qxevZrz58/Tr18/7r77bj755BM2b97Mxo0bOXjwIF26dOGhhx664ngeeOABkpOTSU5O5sUXX+Tb3/72xSHfs7OzWbVqFdu2beOOO+5g69atXH311cXGn5WVxT333MOTTz5JQUHBxblDrkgDd85wJl95HcaEwcqVK1m1ahUpKSkkJib6HY4nVT5p/Ex+Vu5tcnbmlLrdT/WnJa5bunQpY8aMufiLoWHDhoAzbsy4cePYv38/Fy5coF27dmXGMWHCBAD69+/PyZMnOXHiBADDhg2jdu3aAOTl5fHoo4+ydu1a4uLi+OKLL4qt64033uDTTz+92K+Qk5PDli1bWL58ORMmTCAuLo7mzZszaNCgCsWzcuVKXnrpJQCmTJnCD37wg4vbjx07lmrVqtGxY0euvfZaPv/8c9q1a1ds/DfeeCMPPfQQeXl5jBgxomLDPyf2hM1/uvLtjQmD9957j7Vr15KSkkL9+vX9DseziCUNERkC/BGIA/6pqr8KWt8GeBFoAhwDJqvqnorut7Qv+P9r+39O01SQ+m3q893s71Z015f41re+xWOPPcawYcNYtmwZTz/9dJnbeBle/Q9/+ANNmzZl3bp1FBYWUqtWrWLrUlX+/Oc/X5yDo8iSJUs8v4eKDvde3PYlxd+/f3+WL1/O4sWLSUlJ4bHHHmPq1KmeY71EYtGZhjH+U1XeffddNmzYQEpKCgkJCX6HVC4R6dMQkTjgOWAo0AWYICLBwzT+FkhT1R7AM8Avwx3X4GcHU6NOjUuW1ahTg8HPDr7iOgcNGsScOXM4evQowMXmqZycHFq0aAHA9OnTPdU1a9YsAFasWEH9+vWL/TWSk5NDs2bNqFatGunp6RfnHk9ISODUqVMXy91zzz387W9/Iy/PaY774osvOHPmDP3792fWrFkUFBSwf/9+3nnnnQrFc+uttzJz5kwAMjMzuf322y+umzNnDoWFhWzbto3t27dz/fXXlxj/zp07adq0KV//+tf52te+xscff+zpmBWrTksovABnL+9HMiaSVJWlS5eyadMmkpOTK13CgMidafQFtqrqdgARmQkMBzYGlOkCPOY+fwdYEO6givotQnn1VNeuXXnyyScZMGAAcXFx9O7dm9TUVJ5++mnGjBlDgwYNGDRoEDt27Cizrlq1atG7d2/y8vJ48cUXiy3zyCOPMHr0aNLS0hgyZMjFX/09evQgLi6Onj17kpKSwne+8x2ys7O54YYbUFWaNGnCggULGDlyJEuXLqVLly60bt261HmGvcTz5z//ma985Sv85je/oUmTJvz73/++uK5169b07duXkydP8vzzz1OrVq0S41+2bBm/+c1vqFGjBvHx8RcvRb4iIs7ZRs56qN30yusxpgJUlTfffJMdO3aQnJxMnTp1yt4oCkVkaHQReRAYoqpfc19PAW5S1UcDymQBH6rqH0VkFDAPaKyqR4PqmgZMA2jdunWfnTsvnXkvGoZGr4oGDhzIb3/7W5KSrmg05bAo12e95ttQtw10/q/wBmVMMVSV1157jT179jB58uSLfYB+qcjQ6NF0ye33gQEi8gkwANgLFAQXUtUXVDVJVZOK5tc2pkx2Z7jxiaryyiuvsG/fPqZMmeJ7wqioSDVP7QVaBbxu6S67SFX3AaMARCQeGK2qJyIUnylD0XzklVZiD9j8Z7+jMDGmsLCQRYsWcezYMSZPnsxVV13ld0gVFqkzjdVARxFpJyI1gfHAwsACItJYRIri+W+cK6mMCY36XeHUZijMK7usMSFQWFjIggULyMnJYdKkSVUiYUCEkoaq5gOPAq8Dm4DZqrpBRJ4RkWFusYHAZhH5AmgKPFuB/VUwYhPtyv0ZV6/j9Gmc3ByegIwJUFBQwLx588jNzWXChAnUrFnT75BCJmL3aajqEmBJ0LKnAp7PBSo8ol2tWrU4evQojRo1uuy+AFM1qCpHjx4t8Z6UEhUNk57YLTyBGYMzBtu8efMoLCxk/Pjx0TceWwVVrXcDtGzZkj179nD48GG/QzFhVKtWrfLPNZBY1Bk+KSwxGZOfn8/s2bOJi4tj7NixxMXF+R1SyFW5pFGjRg1PQ3SYGJTYA7Y853cUporKy8tj5syZ1K5dm5EjR1bJhAFVMGkYUyK77NaEyYULF5gxYwb16tVj+PDhVKsWTXczhFbVfWfGBKvTCvJz4Zw1XZrQOX/+PBkZGSQmJlb5hAGWNEwsKRpOxM42TIicO3eO9PR0mjZtyrBhw6p8wgBLGibWWBOVCZGzZ8+SlpZGy5Ytuffee2Pmak1LGia2FF12a0wFnDlzhunTp9OuXTvuueeemEkYYEnDxJpEO9MwFXP69GmmT5/Oddddx5133hlTCQPs6ikTaxK7wsnPoTAfqtmfvymfkydPkpaWRvfu3RkwYIDf4fjCzjRMbKle15mUyYYTMeWUk5PD9OnT6d27d8wmDLCkYWKRNVGZcjp+/DipqanceOON9OvXz+9wfGVJw8QemzPclMPRo0dJTU3l1ltv5eabb/Y7HN9Z0jCxp0FPOG5nGqZsR44cYfr06QwYMIAbb7zR73CigvUEmthjN/gZDw4dOkRGRgaDBw+mZ8+efocTNexMw8Seum0g/xScP1p2WROTDhw4QHp6OnfddZcljCCWNEzsseFETCn27dtHRkYGQ4cOpXv37n6HE3UsaZjYZHeGm2Ls3r2bzMxMHnjgAbp06eJ3OFHJ+jRMbGrQE4584HcUJors3LmT2bNnM3LkSDp06OB3OFHLzjRMbLLmKRNgx44dzJ49m9GjR1vCKIOdaZjYVL8b5Gy04UQMW7duZf78+YwZM4a2bdv6HU7UszMNE5tqxEPtFnBqi9+RGB998cUXzJ8/n3HjxlnC8MiSholdDawzPJZt2rSJhQsXMnHiRFq3bu13OJWGJQ0Tu2wMqpi1YcMGFi9ezKRJk2jRooXf4VQqljRM7LLO8Jj06aef8tprrzFlyhSaNWvmdziVTsSShogMEZHNIrJVRH5YzPrWIvKOiHwiIp+KyL2Ris3EqAY2cGGs+eSTT3jrrbeYOnUqTZs29TucSikiSUNE4oDngKFAF2CCiATfOfNjYLaq9gbGA3+NRGwmhtVtCxdy4PwxvyMxEbBmzRqWLVtGcnIyTZo08TucSitSZxp9ga2qul1VLwAzgeFBZRSo5z6vD+yLUGwmVkk1SOxuTVQx4MMPP2TFihUkJyfTqFEjv8Op1CKVNFoAuwNe73GXBXoamCwie4AlwLeKq0hEponIGhFZc/jw4XDEamJJA+sMr+ref/99PvzwQ1JSUmjYsKHf4VR60dQRPgFIVdWWwL1AuohcFp+qvqCqSaqaZKeYpsKsM7xKW758OR999BEpKSkkJib6HU6VEKmksRdoFfC6pbss0FeB2QCquhKoBTSOSHQmdtnAhVWSqvLOO++wfv16UlJSqFevXtkbGU8ilTRWAx1FpJ2I1MTp6F4YVGYXMBhARDrjJA1rfzLhldjdHU6kwO9ITIioKm+//Taff/45KSkpJCQk+B1SlRKRpKGq+cCjwOvAJpyrpDaIyDMiMswt9l/A10VkHTADSFFVjUR8JobVSIDa19hwIlWEqvLGG2+wbds2kpOTqVu3rt8hVTkRG6lNVZfgdHAHLnsq4PlGoF+k4jHmoqI7w+t38jsSUwGqyquvvsrevXuZOnUqtWvX9jukKimaOsKN8Yd1hld6qsqiRYs4cOAAU6ZMsYQRRpY0jLGBCyu1wsJCXn75ZY4dO8akSZOoVauW3yFVaTaRgDE2cGGlVVhYyPz58zlz5gwTJ06kZs2afodU5dmZhjHx7eDCMbhw3O9ITDkUFBQwb948zp07x4QJEyxhRIglDWMuDiey3u9IjEf5+fnMmTOH/Px8xo0bR40aNfwOKWZY0jAG7Ca/SiQvL49Zs2ZRrVo1xo4dS/Xq1soeSXa0jQE3aXzidxSmDHl5ecycOZM6deowcuRIqlWz372RZkfcGLCBCyuBCxcukJWVRUJCgiUMH9lRNwbc4UQ22HAiUer8+fNkZGTQoEEDhg8fbgnDR3bkjQGoUQ+uuhpOb/M7EhPk7NmzpKen07RpUx544AFExO+QYpolDWOK2PSvUSc3N5e0tDRatmzJvffeawkjCljSMKZIYg84bv0a0eLMmTNMnz6d9u3bc88991jCiBLlunpKRK4G4gOXqer2kEZkjF8Se8KONL+jMMCpU6dIT0+nc+fODBw40BJGFPGUNERkCPAv4Bog8NNTIC4McRkTeYnWPBUNTp48SVpaGj169KB///5+h2OCeG2eeg74ORCvqtUCHpYwTNWR0B7OH4ELOX5HErNOnDhBamoqvXv3toQRpbwmjQbA31X1bDiDMcZXUg3qd7P7NXxy/Phxpk+fTt++fenXz6bWiVZek8a/gK+EMxBjooLNreGLo0ePkpqaSr9+/bj55pv9DseUwmtH+M3At0Xkh8CBwBWqaueQpuqwO8Mj7vDhw6Snp3PHHXfQu3dvv8MxZfCaNP7pPoyp2hJ7wI50v6OIGQcPHiQjI4O77rqLHj16+B2O8cBT0lDV6eEOxJiokNgDcj4DLXT6OGLBjkxY9yTk7oI6raHns9BuUth3u3//fjIzMxkyZAjdunUL+/5MaJSYNERkiqqmu88fKqmcqr4YjsCM8UXN+nBVYzi1Dep19Dua8NuRCaumQUGu8zp3p/Mawpo49u7dy4wZM7jvvvvo3Llz2PZjQq+0M40JQNF5+pQSyihgScNULUWd4bGQNNY9+WXCKFKQ6ywPU9LYvXs3M2fOZPjw4Vx33XVh2YcJnxKThqreG/D8jsiEY0wUKJozvPVovyMJv9xd5VteQTt37mT27NmMHDmSDh06hGUfJrw8N9qKSKKITBKRx91/E8uzIxEZIiKbRWSrexVW8Po/iMha9/GFiJwoT/3GhEwsDVxYp3X5llfA9u3bmT17NqNHj7aEUYl5ShoiMgjIBr4N3Ah8C8gWkcEet4/Duat8KNAFmCAiXQLLqOr3VLWXqvYC/gy85PE9GBNaiT1jZ+DCHj+/fFlcHaczPIS2bt3KvHnzGDt2LNdee21I6zaR5fWS278A01R1dtECERmDkwg6edi+L7C1aHBDEZkJDAc2llB+AvBTj7EZE1rx7eHcQcg76cyzUZXVaQZ12jjPw3T11ObNm1m4cCHjx4+nVatWIavX+MNr0mgOzAtaNh/4h8ftWwC7A17vAW4qrqCItAHaAUtLWD8NmAbQunXoT6GNoVoc1O8KJ9ZDkyo+nEV2JnT6DnT6Xliq37RpE4sXL2bixIm0aNEiLPswkeW1TyMd+GbQsm8A4RhHejwwV1WLnXdTVV9Q1SRVTWrSpEkYdm8MsXFneP5Z2PMytBkfluo/++wzFi9ezKRJkyxhVCGl3afxHs4lteAkl4dF5AfAXpwzh6bABx73sxcIPC9t6S4rznguT1DGRFZiDzhexTvD970CDftA7WYhr3rdunW89dZbTJkyhaZNm4a8fuOf0pqngocN8doUVZzVQEcRaYeTLMYDE4MLiUgnnBF1V1ZgX8ZUXIOesHOG31GEV3YmtA39vRgff/wxy5YtY+rUqVhrQNVT2n0aIRs6RFXzReRR4HWcSZteVNUNIvIMsEZVF7pFxwMzVVVLqsuYiEjs7vRpVNXhRM4fg4PvwC2hbWFevXo1K1asIDk5mUaNGoW0bhMdyjXda0Wo6hJgSdCyp4JePx2peIwpVc0GzuP0Dmdypqpm91xodk9Irw774IMP+PDDD0lJSaFBgwYhq9dElyr4E8qYEEmswp3hOzJC2jT1n//8h1WrVpGcnGwJo4qzpGFMSRpU0c7wMzvh5EZoNjQk1S1fvpxPPvmElJQUEhMTQ1KniV4Ra54yptJJ7Ak7Z3oquj5zPW8/+TY5u3Ko37o+g58dTPdJ3cMc4BXKngGtRkNczQpVo6osW7aMTZs2kZKSQnx8fIgCNNHMU9IQkZpACtALuOQvQ1WnhjwqY6JBYg9Y999lFlufuZ5F0xaRl5sHQM7OHBZNWwQQnYkjOxNu/GuFqlBV3nrrLbZt20ZycjJ169YNUXAm2nltnpoOfBc4BWwLehhTNSV0gLMHIO9UqcXefvLtiwmjSF5uHm8/+XY4o7syxz91hkepwJ3uqsrrr7/Ojh07mDp1qiWMGOO1eWoI0E5VT4QxFmOiS7XqUL8LnPgMmtxSYrGcXTnlWu6r7ExoO/GKLyNWVZYsWcL+/fuZOnUqtWrVCnGAJtp5/cvZBVwVzkCMiUqJZQ+TXr91/XIt940WOjcsXuFVU6rKokWLOHjwIFOmTLGEEaNKG0ZkUMDLNOBlEfkjcDCwnKoWO7CgMVWChzGoBj87+JI+DYAadWow+FlPMwdEzqH3nHtPEss/H3dhYSELFy4kJyeHyZMnU7NmxTrRTeVVWvPUv4pZ9j9BrxWwwfFN1ZXYA3bOKrVIUWf320++Tc7OHBJaJHDX/94VfZ3g2Vd2b0ZBQQELFiwgNzeXiRMnUqNGjTAEZyqL0oYRaRfJQIyJSok9PA0n0n1Sd7pP6s7zvZ5n+IvDaXZD6AcBrJCC87D7JRi6tnybFRQwb9488vLymDBhAtWr21X6sc7rzH29RKRV0LJWItIzPGEZEyWuagg16zs3xHmQ0CyB0wdOhzmoK7BviTOeVl3vkyDl5+cze/ZsCgsLGTdunCUMA3jvCM8Ags9Ja+LMs2FM1VaOYdLjr4nn1P7SL9H1RXYmtJ3suXheXh6zZs2ievXqjBkzxhKGuchr0mhdNFVrEVXdBrQNeUTGRJtyjEEV3yye0/uj7EzjQg4ceBNaP+it+IULzJgxg9q1azN69Gji4uLCHKCpTLwmjT0ickPgAvf1vtCHZEyU8XDZbZH4a+Kjr3lq9zxoOhhqJpZZ9Pz582RlZVGvXj1GjBhBtWo2PJ25lNe/iD/gXHL7LRG5V0S+hTNH+O/DF5oxUaJBD+dOag+i8kzD42RL586dIyMjg0aNGjF8+HBLGKZYnhoqVfUfInIC+CrOtK27gf9S1blhjM2Y6JBwHZzdC3mnoUbpg/JFXUd47l44/gm0uK/UYmfPniUjI4MWLVowdOhQRCRCAZrKxnPvlqrOAeaEMRZjolO16lCvM+R8Bo1vLrVo1HWE75wBLUdCXMl3b+fm5pKenk7btm25++67LWGYUnlOGiLSFOgLNAYu/lWp6othiMuY6FJ0Z7iHpHH6wGlUNTq+fLMzoffvSlx95swZ0tLS6NixI4MHD46OmE1U8zo0+gicy263AF2BDUA3YAVgScNUfR4vu60ZX5NqcdU4f/I8ter7PDZTzkY4dwiuHlDs6lOnTpGWlkbXrl0ZMGCAJQzjideerl8AX1HV3sAZ999pwEdhi8yYaJLYo/Jddls0om21yy+ZPXnyJKmpqfTo0YOBAwdawjCelec+jeD+jOmATcBkYkNR0lAts2hUdIarQnZWsVdNnThxgtTUVPr06cPtt9/uQ3CmMvOaNA65fRoA2SJyC9AesLt+TGyo1Riqx3saTiQqOsOPvA/V6zg3JgY4duwYqamp3HTTTdx6660+BWcqM69J4x/Abe7zPwDvAOuAis0ZaUxl4vHO8PhmUXCDX9G9GQHNTkeOHGH69Oncdttt3HTTTT4GZyozr/dp/G/A8zQRWQbUVdVN4QrMmKjTwO0Mbzms1GLx1/jcp1FwAXbNhnvWXFx0+PBh0tPTueOOO+jdu7d/sZlKz/MtnyJSQ0RuF5FxqroL2CUinicHFpEhIrJZRLaKyA9LKDNWRDaKyAYRyfJatzERkXcKNv4KsqrBgrawI7PYYr6faex/Hep1gvi2ABw8eJC0tDTuvPNOSximwrxectsdWAicB1oCs4ABQDIwzsP2ccBzwF3AHmC1iCxU1Y0BZToC/w30U9XjInJ1Od+LMeGzIxO2vQiF55zXuTth1TTnebtLO5t9P9MIGDZk//79ZGZmMnToULp27epfTKbK8Hqm8TfgKVXtBBTNafkuX/ZzlKUvsFVVt6vqBWAmMDyozNeB51T1OICqHvJYtzHht+7JLxNGkYJcZ3mQhGYJ/nWE552C/a9CqzHs3buXzMxM7r//fksYJmS8Jo2uODf3gTPFK6p6BqjtcfsWOONVFdnjLgt0HXCdiPxHRD4QkSHFVSQi00RkjYisOXz4sMfdG1NBubs8L/e1eWr3fGjSn12HcsnKymLYsGF06tTJn1hMleQ1aWQDfQIXiEhfYGsIY6kOdAQGAhOAf4hIYnAhVX1BVZNUNalJkyYh3L0xpajT2vPyOo3qcP7keQouFIQ5qGJkZ5JdaySzZs1i1KhRXHfddZGPwVRpXpPGT4DFIvIzoKaI/DfO4IU/9rj9XpzRcYu0dJcF2gMsVNU8Vd0BfIGTRIzxX89nIa7OpcviajvLg0g1oe7VdTl9MMJnG2cPsH3nfua8d4QHH3yQ9u3bR3b/JiZ4Shqq+gowBGiC05fRBhilqm943M9qoKOItBORmsB4nI71QAtwzjIQkcY4zVXbMSYatJsEfV+AOm0AcW70S+x1WSd4ET86w7d8kMG8/SMYO3Yc7dq1i+i+Tewoz9DonwCPXMlOVDVfRB4FXse5i/xFVd0gIs8Aa1R1obvubhHZCBQAj6vq0SvZnzFh0W7Sl0ki/yy81se5qqqYxBHpoUQ2b97MwvePMeG+PrRs0yZi+zWxJ2KzxavqEmBJ0LKnAp4r8Jj7MCa6Va8N/bJg6d3QpN/FeyKKxDeL3FAiGzduZMnihUxq8wrNezwTkX2a2GXzORpzpRr0gs6Pw8opUHhpp3ek5gpfv349r776KpNvOkvzzoOcCaOMCSNLGsZUROf/gmo1nDvFA0RiePS1a9fy5ptvMmXyZK454W0ecGMqypKGMRUh1eDm6fDFn+Do6ouLw90R/vHHH/POO+8wdepUro7LduJomBS2/RlTxOswIg2B7wO9gPjAdaraP/RhGVOJ1G0FSX+B9yfBkI+hRnxYO8JXrVrF+++/T3JyMg0bNoQ1z0DbyZeMaGtMuHhtAM0CrgJmA7nhC8eYSqr1GNi7GD5+DG56IWxzaqxcuZJVq1aRnJxMgwYNoDAPds2Cu/4T8n0ZUxyvSeNWoImqng9nMMZUakl/gld7w+4FxF9zP2cOnkFVQzaV6ooVK/jkk09ISUmhfv36zsIDb0HddpDQIST7MKYsXvs0PsW5i9sYU5Ia9eCWdFj9MNX1MDXq1uDssbMhqfrdd99l7dq1lyYMuGREW2MiweuZxlLgNRH5N3AgcIWqvhjyqIyprJrcCh0ehpUpxF8zjNMHTlOnUZ2ytyuBqvLOO+/w+eefk5KSQnx8QJdi/hnY+wrc8PsQBG6MN16Txu04Y0PdFbRcAUsaxgTq9mN483YS6p/g9P7TXN31yqaGUVXefPNNtm/fTnJyMnXrBs15tudlaHwL1LKpZ0zkeJ3u9Y5wB2JMlVGtOtyaQXz1n3Bq6ya489pyV6GqvPbaa+zevZvk5GRq1y5mFgJrmjI+KLFPQwJ670SkWkmPyIRpTCWT0J7465M4vfrfUHCu7PIBVJXFixezb98+pk6dWnzCOHcYDv8HWo4ITbzGeFTal35OwPN8nBn7Ah9Fy4wxxYjvdDOnTzeDtT/yvE1hYSELFy7k8OHDTJ48mVq1ahVfcNdsaH4v1Igvfr0xYVJa81Tg/JA2zrIx5RTfLJ793AK7n4DmQ6FZcJfgpQoLC3n55Zc5efIkkyZNombNmiUXzs6Erl6nszEmdEpMGqq6O+D5zsiEY0zVkdAsgdOH8uDmVFiZDEPXQq3GxZYtKChg/vz5nD17lokTJ1KjRo2SKz61DU5tLTMJGRMO1idhTJhcHOn2msHQZjysmgaql5UrKChg7ty5XLhwgQkTJpSeMACys6D1WGegRGMizJKGMWFyyZwaPZ+F09th+6VXqOfn5zN79mwAxo0bR/XqZVzQqAo77aop4x9LGsaESa3EWuSfyyfvbB7EXQW3ZsLaH8LJLQDk5eUxc+ZMqlevzoMPPkhcXFzZlR7/2BlvqvHNYY7emOJ5ShoiMlxEbHYXY8pBRC6djCmxK3R7ClZO5sK5M2RlZVGnTh1Gjx7tLWGAM71sm4k2oq3xjdczjWeA/SLyFxG5KZwBGVOVJDRLuHRejese5Xy1xmT+8zckJiYyYsQIqlXz+N+wsAB2zbSmKeMrT3+tqtoTuBM4C8wTkc0i8mMRaRvO4Iyp7IKnfT13/jwZ2ffTRDcy7OaG3hMGwKF3oFYzqN8pDJEa443nv1hVXaeqjwOtgG8CY4BtIrJcRCbZ3eHGXC6wM/zs2bOkp6fTvGVb7hsxGflgKlzIKaOGANmZ0G5ymCI1xptyfdGLSHvgKeBvQC33+T+AR4G5IY/OmEqu6EwjNzeXtLQ02rRpw5AhQ5BWw6DZPbDmW94qyj8Luxc4l+4a4yOvHeHfFJEPgFVAU2CKql6vqs+qajowGLg7jHEaUynFN4vn+MHjpKam0qFDB+66664vJ2W64Xdw9EPYOavsivYugkZJULtZeAM2pgxezzSGAr8DmqvqI6r6QeBKVc0FRpVWgYgMcftCtorID4tZnyIih0Vkrfv4mtc3YUy0imsUx+aGm+natSuDBg26dBa/6nWhX5ZztnFmd8mVgI1oa6KG16SxTFXnBE/3KiKPFT1X1TdK2lhE4oDncJJPF2CCiHQppugsVe3lPv7pMTZjolJOTg5vrH+D/JX5LLtjGX9s90fWZ66/tFDDPtDpMVg5xbk6qjjnj8GhZdCq1N9lxkSE16TxVAnLvY6Y1hfYqqrbVfUCMBMY7nFbYyqdEydO8MJfXuD80vPocgWFnJ05LJq26PLE0flxQOHz3xZf2a45Tv9HjXphj9uYspSaNERkkIgMAqqLyB1Fr93H14BTHvfTAgg8/97jLgs2WkQ+FZG5ItLKY93GRJVjx46RmppK4YpCCpcXXrIuLzePt598+9INqsU5c4tv+h0c+/jyCq1pykSRsu7y/pf771VcOq2r4swV7vHSD08WATNU9byI/D9gOjAouJCITAOmAbRu3TqEuzem4o4cOUJ6ejr9+/fnlcdeKbZMzq5iLrOt2xr6/BHenwhDPobq7rziZ3bCyY3QbGgYozbGu1LPNFS1naq2AzKLnruPa1X1VlVd6HE/e3Hu7yjS0l0WuK+jAX0m/wT6lBDTC6qapKpJTZo08bh7Y8Lv0KFDpKWlcccdd9CnTx/qt65fbLmSltN2AjRMgk++/+Wy7Cxo9SDElTK3hjER5PWO8KkV3M9qoKOItBORmsB44JKEIyKB1xIOAzZVcJ/GRMyBAwdIT0/nrrvuolevXgAMfnYwNepcOnx5jTo1GPzs4JIrSnoO9i2Bj78PC9rAuh/BngXOmFPGRIESm6dEZJOqdnaf78ZpkrqMqpbZRqSq+SLyKPA6EAe8qKobROQZYI17xvJtERmGM43sMSClvG/GGD/s27ePrKwshg4dSteuX0542X1SdwDefvJtcnblUL91fQY/O/ji8mLVrA9tp8KGn3+57NxBZy4OgHbWt2H8JVrMpDAAInKbqq5wnw8oqQJVfTdMsZUpKSlJ16xZ49fujWHPnj3MnDmT+++/n06dQjQm1IK2kFvMZJl12sCI7NDsw8Q0EflIVZOuZNvSpntdEfDct8RgTLTatWsXs2bNYsSIEXTs2DF0FefuKt9yYyKotOapZ7xUoKol3cNhTJWVnZ3NnDlzGDVqFO3btw9t5XVal3CmYVcLGv+Vdsmt3SdhTDG2bdvGSy+9xJgxY2jbtm3od9DzWacPoyD3y2VxdZzlxvistOapr0QyEGMqgy1btrBgwQLGjRsXvvuEijq71z3pNEnVae0kDOsEN1GgtOaptqqa7T6/tqRyqro9DHEZE3U+//xzXnnlFSZMmEDLli3Du7N2kyxJmKhUWvPUeiDBfb4V55Lb4ImJFecSWmOqtA0bNvDqq68yceJEmjdv7nc4xvimtOaphIDnNiufiVnr16/njTfeYPLkyVxzzTV+h2OMr8oae+oSItICaA7sVdV94QnJmOixdu1ali5dypQpU7j66qv9DscY33mdua+1iLwH7AQWA7tE5D0RaRPW6Izx0UcffcQ777zD1KlTLWEY4/La7DQd+Aior6pXA4nAGne5MVXOqlWreO+990hOTqZx48Z+h2NM1PDaPNUHuFtV8wBU9bSIPAEcDVtkxvhk5cqVrFq1ipSUFBITE/0Ox5io4vVM4wOc2fcCJQErQxuOMf567733WLNmjSUMY0rgdRiRbcASEVmMMwNfK+BeICu84RkTGarKu+++y4YNG0hJSSEhIaHsjYyJQeUZRuQl99+rgfPAfKBWOIIyJpJUlaVLl/LFF1+QnJxMfHy83yEZE7VsGBET01SVN998kx07dpCcnEydOnX8DsmYqFbe+zQSgMYE3Bluw4iYykpVee2119izZw9Tp06ldu3afodkTNTzlDREpAuQCfTky+FEimZvsmFETKWjqrzyyiscOnSIKVOmUKuWtbQa44XXq6f+CrwDNAROAg2AvwPJYYrLmLApLCxk4cKFHDlyhMmTJ1vCMKYcvDZP9QTuUtU8ERFVzRGRx4HPgIzwhWdMaBUWFrJgwQJOnz7NpEmTqFmzpt8hGVOpeD3TOAfUcJ8fEZHW7raNwhKVMWFQUFDAvHnzyM3NZcKECZYwjLkCXpPGe8BY9/lc4FXgXWBpOIIyJtTy8/OZO3cu+fn5jB8/nho1apS9kTHmMp6ap1R1bMDLHwEbgHggLRxBGRNK+fn5zJ49m7i4OMaOHUtcnF27YcyVKu8lt4LTJJWhqlpWeWP8lpeXx8yZM6lduzYjR460hGFMBXkdGj1RRNKBs8BB4KyIpItIw7BGZ0wFXLhwgaysLOLj4xk1apQlDGNCwGufxr+B2kBvnGap3sBVwItedyQiQ0Rks4hsFZEfllJutIioiCR5rduYYOfPnyczM5PExESGDx9OtWo2+aQxoeC1eWoQcI2qnnVfbxKRFMDT7H0iEgc8B9wF7AFWi8hCVd0YVC4B+A7woce4jLnMuXPnyMjI4JprruG+++7DaVU1xoSC159fnwNtg5a1BjZ73L4vsFVVt6vqBWAmMLyYcj8H/hfnEl9jyu3s2bOkpaXRokULSxjGhEFpQ6M/FPDybeANt1+jaGj0yUC6x/20cLcrsge4KWh/NwCtVHWxe+NgSXFNA6YBtG7d2uPuTSw4c+YM6enptG/fnjvvvNMShjFhUFrz1JSg11uBW9wHOHNs3EIIiEg14PdASlllVfUF4AWApKQku4LLAHD69GnS0tLo1KkTd9xxhyUMY8KktKHR7wjhfvZy6fwcLd1lRRKAbsAy9z/7NcBCERmmqmtCGIepgk6ePElaWhrdu3dnwIABfodjTJXm+T4NEWkAPIDT1LQXeEVVj3ncfDXQUUTauduOByYWrVTVHJwh14v2tQz4viUMU5acnBymT5/ODTfcwG233eZ3OMZUeV7v07gFpznqYaAH8P+Are7yMqlqPvAo8DqwCZitqhtE5BkRGXZFkZuYd/z4cVJTU7nxxhstYRgTIV7PNP4PeERVZxYtEJFxwJ+AG71UoKpLgCVBy54qoexAj3GZGHXs2DHS0tLo168fN97o6U/QGBMCXi+5vQ6YHbRsLtAhtOEYU7YjR46QmppK//79LWEYE2Fek8YWnH6IQGNwmqyMiZhDhw4xffp0Bg8ezA033OB3OMbEHK/NU98FXhGRbwM7cW706wjcH56wjLncgQMHyMzM5O6776Z79+5+h2NMTCozabgj2x4AOgF3A82BRcCSclw9ZUyF7Nu3j6ysLO699166dOnidzjGxKwyk4aqqoisBxJU1aZ2NRG3e/duZs6cybBhw7j++uv9DseYmOa1T+MTnM5wYyJq586dzJw5k5EjR1rCMCYKeO3TWAa8JiKpOGNIXRy+Q1U9D49uTHns2LGDuXPnMnr0aK699lq/wzHG4D1p9AN2AMFjNCjlmFPDGK+2bt3K/PnzGTNmDG3btvU7HGOMy+sc4aEch8qYUn3xxRe8/PLLjBs3zkYyNibKlGfsqUTgPpyrp/YBi1X1RHjCMrFq06ZNLF68mIkTJ9KiRQu/wzHGBPE69tQgIBv4Ns6wId8CskVkcPhCM7Fmw4YNLF68mEmTJlnCMCZKeT3T+AswTVUvDiUiImNwpnDtFI7ATGz59NNPefPNN5kyZQpNmzb1OxxjTAm8XnLbHJgXtGw+zrwXxlTIJ598wltvvcXUqVMtYRgT5bwmjXTgm0HLvgGkhTYcE2vWrFnDsmXLSE5OpkmTJn6HY4wpg9fmqd7AwyLyA5xJlFoAVwMfisjyokKq2j/0IZqq6sMPP2TlypUkJyfTsGFDv8MxxnjgNWn8w30YExLvv/8+a9asISUlhcTERL/DMcZ45PU+jenhDsTEjuXLl7Nu3TpSUlKoV6+e3+EYY8rB830axlSUqrJs2TI2btxISkoKCQkJfodkjCknSxomIlSVt99+my1btpCSkkLdunX9DskYcwUsaZiwU1XeeOMNsrOzSU5Opk6dOn6HZIy5QpY0TFipKq+++ip79+5l6tSp1K5d2++QjDEV4HUYkatE5FkR2S4iOe6yu0Xk0fCGZyozVWXRokUcOHCAKVOmWMIwpgrwenPfH4BuwCS+nEtjA84NfsZcprCwkJdffpljx44xadIkatWq5XdIxpgQ8No8NRLooKpnRKQQQFX3ioiNKmcuU1hYyIIFCzh9+jQTJ06kZs2afodkjAkRr2caFwhKMCLSBDjqdUciMkRENovIVhH5YTHrHxaR9SKyVkRWiEgXr3Wb6FFQUMC8efM4e/YsEyZMsIRhTBXjNWnMAaaLSDsAEWmGM/LtTC8bi0gczoi4Q4EuwIRikkKWqnZX1V7Ar4Hfe4zNRIn8/HzmzJlDfn4+48aNo0aNGn6HZIwJMa9J40c4072uBxKBLTgTMf3M4/Z9ga2qul1VL+Akm+GBBVT1ZMDLugTMQ26iX15eHrNmzaJatWqMHTuW6tXtwjxjqiKvw4hcAL4HfM9tljqiquX5Um8B7A54vQe4KbiQiHwTeAyoCQwqriIRmQZMA2wq0CiRl5fHzJkzqVOnDiNHjqRaNa+/RYwxlY3XS26vLXoACUC7gNcho6rPqWp74AngxyWUeUFVk1Q1yYbS9t+FCxfIysoiISHBEoYxMcBrG8JWnOYiCVhWdKYR52H7vUCrgNct3WUlmQn8zWNsxifnz58nMzOTxo0b88ADDyAiZW9kjKnUPP0sVNVqqhrn/lsNZya/F4ApHvezGugoIu1EpCYwHlgYWEBEOga8vA+n38REqbNnz5Kenk7Tpk0tYRgTQ66ot1JVD4jId4EvgCwP5fPdu8dfxzkzeVFVN4jIM8AaVV0IPCoidwJ5wHEg+UpiM+GXm5tLRkYGrVu35p577rGEYUwMqcglLtcDnkeeU9UlwJKgZU8FPP9OBWIxEXLmzBnS09Pp0KEDgwcPtoRhTIzxlDRE5D0uvQS2DtAVeCYcQZnodOrUKdLT0+ncuTMDBw60hGFMDPJ6pvHPoNdngHWqav0OMeLkyZOkpaXRo0cP+ve3qeCNiVVlJg33bu5BwDRVPR/+kEy0OXHiBGlpafTp04d+/fr5HY4xxkdlJg1VLRCRu4HCCMRjoszx48dJS0vjpptu4uabb/Y7HGOMz8ozNPrPRMQGE4ohR48eJTU1lX79+lnCMMYAZSQNEZngPv0W8DhwSkR2i8iuokfYIzS+OHz4MNOnT2fgwIEkJSX5HY4xJkqU1Tz1d2AGMDkCsZgocfDgQTIyMrjzzjvp2bOn3+EYY6JIWUlDAFT13QjEYqLA/v37yczMZMiQIXTr1s3vcIwxUaaspBEnIndw6ZhTl1DVpaENyfhl7969zJgxg/vuu4/OnTv7HY4xJgqVlTSuAv5FyUlDgZCOdGv8sXv3bmbOnMmwYcO4/vrr/Q7HGBOlykoaZ1TVkkIVt3PnTmbPns3IkSPp0KGD3+EYY6KYTa8W47Zv3868efMYPXo0115rvw+MMaXz1BFuqqatW7cyf/58xo4dS5s2bfwOxxhTCZSaNFQ1IVKBmMjavHkzCxcuZPz48bRq1arsDYwxBmueikmbNm1i8eLFTJw4kRYtWvgdjjGmErGkEWM+++wzXnvtNSZNmkSzZs38DscYU8lY0ogh69at46233mLKlCk0bdrU73CMMZWQJY0Y8fHHH7Ns2TKmTp1KkyZN/A7HGFNJWdKIAatXr2bFihUkJyfTqFEjv8MxxlRiljSquA8++IAPP/yQlJQUGjRo4Hc4xphKzpJGFfaf//yHjz76iOTkZBITE/0OxxhTBVjSqKKWL1/Op59+SkpKCvXq1fM7HGNMFWFJo4pRVZYtW8amTZtISUkhPj7e75CMMVWI1+leK0xEhojIZhHZKiI/LGb9YyKyUUQ+FZG3RcTGtSgnVeWtt95i8+bNJCcnW8IwxoRcRJKGiMQBzwFDgS7ABBHpElTsEyBJVXsAc4FfRyK2qkJVef3119mxYwdTp06lbt26fodkjKmCInWm0RfYqqrbVfUCMBMYHlhAVd9R1Vz35QdAywjFVumpKkuWLGHPnj1MnTqVOnXq+B2SMaaKilTSaAHsDni9x11Wkq8Cr4Y1oipCVVm0aBEHDx5kypQp1KpVy++QjDFVWNR1hIvIZCAJGFDC+mnANIDWrVtHMLLoU1hYyMKFC8nJyWHy5MnUrFnT75CMMVVcpM409gKB42+3dJddQkTuBJ4Ehqnq+eIqUtUXVDVJVZNieTiMgoIC5s+fz6lTp5g4caIlDGNMREQqaawGOopIOxGpCYwHFgYWEJHewN9xEsahCMVVKRUUFDBv3jzOnTvHhAkTqFGjht8hGWNiRESShqrmA48CrwObgNmqukFEnhGRYW6x3wDxwBwRWSsiC0uoLqbl5+cze/ZsCgsLGTduHNWrR10LozGmCovYN46qLgGWBC17KuD5nZGKpbLKy8tj9uzZ1KxZk1GjRhEXF+d3SMaYGGM/UyuJCxcuMHPmTOLj4xkxYgTVqkXsvkxjjLnIkkYlcP78eWbMmEFiYiLDhg2zhGGM8Y0ljSh37tw5MjMzufrqq7n//vsREb9DMsbEMEsaUezs2bNkZGTQokULhg4dagnDGOM7SxpRKjc3l/T0dNq2bcvdd99tCcMYExUsaUShM2fOkJaWRseOHRk8eLAlDGNM1LCkEWVOnTpFWloaXbt2ZcCAAZYwjDFRxZJGFDl58iTTp0+nV69e3H777X6HY4wxl7GkESVOnDhBWloaSUlJ3HrrrX6HY4wxxbKkEQWOHTtGWloat9xyCzfddJPf4RhjTIksafjsyJEjpKenc/vtt5OUlOR3OMYYUypLGj46fPgw6enp3HHHHfTu3dvvcIwxpkyWNHxy8OBBMjIyuOuuu+jRo4ff4RhjjCeWNHywf/9+MjMzGTp0KF27dvU7HGOM8cySRoTt3buXGTNmcN9999G5c2e/wzHGmHKxpBFBu3btYtasWQwfPpzrrrvO73CMMabcLGlESHZ2NnPmzGHUqFG0b9/e73CMMeaKWNKIgO3btzNv3jwefPBB2rVr53c4xhhzxSxphNmWLVtYsGABY8eOpU2bNn6HY4wxFWJJI4w2b97MwoULGT9+PK1atfI7HGOMqTBLGmGyceNGlixZwqRJk2jevLnf4RhjTEhY0giD9evX88YbbzB58mSuueYav8MxxpiQsaQRYmvXrmXp0qVMmTKFq6++2u9wjDEmpCxphNDHH3/MsmXLmDp1Ko0bN/Y7HGOMCblqkdqRiAwRkc0islVEfljM+v4i8rGI5IvIg5GKK1RWrVrF8uXLSU5OtoRhjKmyIpI0RCQOeA4YCnQBJohIl6Biu4AUICsSMYXSypUrWblyJcnJyTRq1MjvcIwxJmwi1TzVF9iqqtsBRGQmMBzYWFRAVbPddYURiikkVqxYwSeffEJKSgr169f3OxxjjAmrSDVPtQB2B7ze4y4rNxGZJiJrRGTN4cOHQxLclXr33XdZu3atJQxjTMyIWJ9GqKjqC6qapKpJTZo08SsGli5dyoYNG0hJSSEhIcGXOIwxJtIi1Ty1Fwi8Jbqlu6zSUVXefPNNtm/fTnJyMnXr1vU7JGOMiZhInWmsBjqKSDsRqQmMBxZGaN8ho6q89tprZGdnW8IwxsSkiCQNVc0HHgVeBzYBs1V1g4g8IyLDAETkRhHZA4wB/i4iGyIRm1eqyuLFi9m3bx9Tp06ldu3afodkjDERF7Gb+1R1CbAkaNlTAc9X4zRbRZ3CwkIWLVrEsWPHmDx5MldddZXfIRljjC/sjvAyFBYW8vLLL3Py5EkmTZpEzZo1/Q7JGGN8Y0mjFAUFBcyfP5+zZ88yceJEatSo4XdIxhjjK0saJSgoKGDu3LkUFBQwYcIEqle3Q2WMMZXuPo1IyM/PZ/bs2QCMGzfOEoYxxrgsaQTJy8tj5syZVK9enQcffJC4uDi/QzLGmKhhP6EDXLhwgRkzZpCQkMCIESOoVs1yqjHGBLKk4Tp//jxZWVk0bNiQBx54wBKGMcYUw5IGcO7cOTIzM2natCn33XcfIuJ3SMYYE5ViPmmcPXuWjIwMWrZsyZAhQyxhGGNMKWI6aeTm5pKenk67du246667LGEYY0wZYjZpnD59mrS0NK6//noGDRpkCcMYYzyIyaRx6tQp0tLS6NatG/3797eEYYwxHsVc0sjJySEtLY3evXtz2223+R2OMcZUKjGVNE6cOMH06dPp27cvt9xyi9/hGGNMpRMzSePYsWOkpaVx66230rdvX7/DMcaYSikmksaRI0dIT0+nf//+9OnTx+9wjDGm0qrySePQoUNkZGQwaNAgevXq5Xc4xhhTqVXppHHgwAEyMzO5++676d69u9/hGGNMpVdlk8a+ffvIyspi6NChdO3a1e9wjDGmSqiSSWPPnj3MnDmT+++/n06dOvkdjjHGVBlVLmns2rWLWbNmMWLECDp27Oh3OMYYU6VUqaSRnZ3NnDlzGDVqFO3bt/c7HGOMqXKqTNLYtm0bL730EmPGjKFt27Z+h2OMMVVSlUgaW7ZsYcGCBYwbN47WrVv7HY4xxlRZEZueTkSGiMhmEdkqIj8sZv1VIjLLXf+hiLT1Uu/nn3/Oyy+/zIQJEyxhGGNMmEUkaYhIHPAcMBToAkwQkS5Bxb4KHFfVDsAfgP8tq96zZ8/yyiuvMHHiRFq2bBnqsI0xxgSJ1JlGX2Crqm5X1QvATGB4UJnhwHT3+VxgsJQxZvnJkyeZPHkyzZs3D3nAxhhjLhepPo0WwO6A13uAm0oqo6r5IpIDNAKOBBYSkWnANPfl+WbNmn0WlojLpzFBccZoDBAdcVgMX4qGOKIhBoiOOKIhBoDrr3TDStcRrqovAC8AiMgaVU3yOaSoiCMaYoiWOCyG6IojGmKIljiiIYaiOK5020g1T+0FWgW8bukuK7aMiFQH6gNHIxKdMcYYTyKVNFYDHUWknYjUBMYDC4PKLASS3ecPAktVVSMUnzHGGA8i0jzl9lE8CrwOxAEvquoGEXkGWKOqC4F/AekishU4hpNYyvJC2IIun2iIIxpigOiIw2L4UjTEEQ0xQHTEEQ0xQAXiEPsxb4wxxquI3dxnjDGm8rOkYYwxxrNKkTTCNQRJiGPoLyIfi0i+iDwY6v2XI47HRGSjiHwqIm+LSBsfYnhYRNaLyFoRWVHM3f8RiSOg3GgRUREJ+aWOHo5Fiogcdo/FWhH5Wqhj8BKHW2as+7exQUSyIh2DiPwh4Dh8ISInQh2Dxzhai8g7IvKJ+//kXh9iaOP+//xURJaJSMiHtBCRF0XkkIgUey+bOP7kxvipiNzgqWJVjeoHTsf5NuBaoCawDugSVOYR4Hn3+Xhglg8xtAV6AGnAgz4eizuAOu7zb/h0LOoFPB8GvObHsXDLJQDLgQ+AJB+ORQrwl3D8PZQzjo7AJ0AD9/XVfnweAeW/hXNBjB/H4gXgG+7zLkC2DzHMAZLd54OA9DAci/7ADcBnJay/F3gVEOBm4EMv9VaGM42wDEES6hhUNVtVPwUKQ7jfK4njHVXNdV9+gHNPTKRjOBnwsi4QjqstvPxdAPwcZxyzcz7GEG5e4vg68JyqHgdQ1UM+xBBoAjAjxDF4jUOBeu7z+sA+H2LoAix1n79TzPoKU9XlOFeilmQ4kKaOD4BEEWlWVr2VIWkUNwRJi5LKqGo+UDQESSRjiITyxvFVnF8SEY9BRL4pItuAXwPfDnEMnuJwT7dbqeriMOzfUwyu0e7p/1wRaVXM+kjEcR1wnYj8R0Q+EJEhPsQAOE0zQDu+/NKMdBxPA5NFZA+wBOesJ9IxrANGuc9HAgkiEsrvLC+u6HutMiQNcwVEZDKQBPzGj/2r6nOq2h54AvhxpPcvItWA3wP/Fel9B1kEtFXVHsCbfHlGHGnVcZqoBuL8yv+HiCT6FMt4YK6qFvi0/wlAqqq2xGmiSXf/XiLp+8AAEfkEGIAzIoZfx6NcKkPSiIYhSLzEEAme4hCRO4EngWGqet6PGALMBEaEOAYvcSQA3YBlIpKN02a7MMSd4WUeC1U9GvAZ/BPoE8L9e44D51fkQlXNU9UdwBc4SSSSMRQZT3iaprzG8VVgNoCqrgRq4QwkGLEYVHWfqo5S1d44/1dR1RMhjMGLK/teC3XnSxg6c6oD23FOZ4s6lboGlfkml3aEz450DAFlUwlfR7iXY9EbpxOuo48xdAx4/gDOXf8RjyOo/DJC3xHu5Vg0C3g+EvjAp89kCDDdfd4Yp1miUaQ/D6ATkI17Y7FPx+JVIMV93hmnTyNk8XiMoTFQzX3+LPBMmI5HW0ruCL+PSzvCV3mqMxyBhuGN34vzy2gb8KS77BmcX9Lg/FKYA2wFVgHX+hDDjTi/5s7gnOVs8OlYvAUcBNa6j4U+xPBHYIO7/3eK+/KIRBxBZZcR4qTh8Vj80j0W69xj0cmnvwvBaa7bCKwHxvvxeeD0J/wqHMegHMeiC/Af9zNZC9ztQwwPAlvcMv8ErgpDDDOA/UCe+930VeBh4OGAv4nn3BjXe/3/YcOIGGOM8awy9GkYY4yJEpY0jDHGeGZJwxhjjGeWNIwxxnhmScMYY4xnljRMpSYiqSLyC/f57SKyOUL7VRHpEIH9tHX3dUWzbJYWp4hMEpE3iisrIs+LyE+uLGpTlVnSMGEnItkiclZETovIQfeLPj7U+1HV91T1eg/xpIjIilDvv7JR1UxVvbuEdQ+r6s8BRGSgO06TMZY0TMQ8oKrxOEM1J1HMeFRX+mu6MovF92wqN0saJqJUdS/O0AXd4GKTyDdFZAvOHbKIyP3uRD0nROR9EelRtL2I9BZnsqtTIjILZzSAonWX/CIWkVYi8pI7CdJREfmLiHQGngducc98TrhlrxKR34rILvds6HkRqR1Q1+Misl9E9onIQ6W9R3dSnV+KyCoROSkiL4tIQ3ddUXPTV0VkF7BURKqJyI9FZKc7aU6aiNQPqvYhd9/7ReT7AfvqKyIr3WO1332PNYO2vVdEtovIERH5TdHgfKWdcRU1+4lIXffzau4er9Mi0lxEcgNHZRWRG9zjXKO0Y2MqP0saJqLcocHvxZkQqMgI4Cagi4j0Bl4E/h/O8PZ/xxlo8Cr3y3ABkA40xBk6ZnQJ+4kDXgF24oy/0wKYqaqbcIZSWKmq8aqa6G7yK5zhw3sBHdzyT7l1DcEZlfQunEH+7vTwVqcCDwHNgHzgT0HrB+CMe3QPzkRNKTgTaF0LxAN/CSp/h7vvu4En3EEpwRkZ9Xs4YxndAgzGmZQs0Eics7sbcOZQKDXpBVLVM8BQYJ97vOJVdR/OsCxjA4pOwTm+eV7rNpVUOMeAsYc9VBWcAepOAydwvsT/CtR21ykwKKDs34CfB22/GedLtj9Bg8sB7wO/cJ8PBPa4z28BDgPVi4knBVgR8FpwxgxrH7DsFmCH+/xFAsZLwkkuCnQo4f0uCyrfBbiAM6NbW3fbawPWvw08EvD6epzxgqoHlO8UsP7XwL9K2Pd3gfkBrxUYEvD6EeDtEo7DxfeEM/DmZcc1oOw44D/u8zjgANDX7781e4T/Ye2pJlJGqOpbJawLnAimDZAsIoET49QEmuN8qe1V95vKtbOEOlsBO9WZlKssTYA6wEfy5YSPgvNliLvvjzzsM1Dge9oJ1ODS4bcD1zcPqnMnTsJoWkp93QFE5DqcgQiT3PdQPSjW4rZt7iH+srwMPC8i7XCSXI6qrgpBvSbKWfOUiQaBSWA38KyqJgY86qhq0YidLUQumcq3dQl17gZal9DRHDxK5xHgLM5ovEX7rK9Oxz3ufgPnHShpn4GCy+e5+ykuhn04yTKwfD7OaMUl1Vc0RenfgM9xhqOvB/wIJ+GVFkt5pze9bFRTVT2HMyfFZJymqfRy1mkqKUsaJtr8A3hYRG4SR10RuU9EEoCVOF+m3xaRGiIyCmc+5uKswvmy/5VbRy0R6eeuOwi0LOowVtVCd79/EJGrAUSkhYjc45afDaSISBcRqQP81MP7mBxQ/hlKn6luBvA9EWnnXor8P8CsoLOkn4hIHRHpCnwFmOUuTwBOAqdFpBPwjWLqf1xEGrj9Sd8J2Narg0CjYjrn03CauIZhSSNmWNIwUUVV1wBfx+kIPo4zR0qKu+4CzrzKKcAxnHb1l0qopwBnAqgOwC6c+QTGuauX4sxxcUBEin79P+Hu6wMROYkzL8n1bl2vAv/nbrcVb3Nbp+P0CxzAucKrtHnSX3TLLwd2AOe4fN7qd919vw38VlWLbsr7PjAROIWT+IpLCC/jNFmtBRYD//IQ/0Wq+jlOYtvuXqXV3F3+H6AQ+FhVvTTZmSrA5tMwJsREZBmQoar/9DuWcBORpUBWLLxX47COcGPMFRGRG/nyMl4TI6x5yhhTbiIyHacJ77uqesrveEzkWPOUMcYYz+xMwxhjjGeWNIwxxnhmScMYY4xnljSMMcZ4ZknDGGOMZ/8ftUaBDFRaPzYAAAAASUVORK5CYII=",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAJNCAYAAADqP9XhAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAkO1JREFUeJztnQWYVOXbxp/tJTbo7u7uUBEFEURKGkQEFcXAAlHCwhb/gKKIoIQ0iLQgKEh3d3cuS27Od93vcvab3Z2NmZ2Zc87M/buuYZkzJ545c2be+zz1+lgsFosQQgghhBgEX70NIIQQQgixhuKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBAn8Ndff0mfPn2kbNmyEhoaKkFBQVKgQAF57LHH5Ntvv5UrV6445TgPP/yw+Pj4yJo1a5IsHzFihFqOv9ZMnjxZLX/22WfFTKT2fjwBfHZ4b/gsCSG2oTghJBNcvXpVCZDHH39cCYGYmBh55JFHpEOHDlKhQgVZv369DBo0SEqWLCmbNm3S21xDwMHZuZhVgBKSFv5pvkoISZWbN29K48aN5dChQ1K+fHn56aefpEmTJknWiYqKkl9//VWGDx8uFy5ccJktr7zyinTp0kVy584tnoCnvR9CiH1QnBDiIAMHDlTCpHjx4vLff/9Jzpw5U6yD8E7//v2lbdu2EhER4TJbMIh70kDuae+HEGIfDOsQ4gDHjx+X6dOnq/9/8803NoWJNfny5ZNy5colPr9165ZMmDBB2rdvL2XKlJFs2bKpR5UqVWTo0KF2C5mM5Ghcu3ZNXn75ZSlatKgSTcWKFZM33nhDbty4kWbo5e7duzJs2DAVpsqaNasSYxqbN2+Wd955R+rWrSv58+eXwMBA9V7btGkjK1euTLFf7A9hL/DPP/+oY2gP6/2m936WL18urVu3lrx586pjFixYUDp37ixbt25NN1dn586d6rxD/OA8VKxYUb7++muxd4J2axtPnTolvXr1UnlGwcHBKvcIy+/duyf2cvDgQZW/hM8H9uHaevTRR2XWrFkp1sU5w7oAHjrr88mwGTEz9JwQ4gCLFi2SuLg4CQ8Pl6eeesru7Xft2qU8Knny5FGipVatWkokbNu2TT799FM1EG3cuFFy5crlFHux73r16imBYj1Qjx49WpYuXSpr165VtiTn/v37av39+/dL06ZNpVq1amofGu+9956sXr1aKlWqpN4DBNaxY8fU+cED+3/ttdcS12/ZsqUavCEuIGLwXCOjnpIPPvhAPv74Y/UeGjZsqMTWgQMH1DmbO3euCq8999xzNrfFcSEmS5UqpXKFEGpbt26dvPXWW3LmzBllr72cOHFCvXd/f391jiBIcE5GjhypBBoeeM8ZYfHixdKxY0d13nFdQERdvnxZCbm///5b2T9x4sTE9bEurhN47vCeEGbUQKiRENNiIYTYTc+ePXGbbWnWrJlD2585c8aycuVKS1xcXJLld+7csfTq1Uvte8CAASm2e+ihh9Rrq1evTrJ8+PDhajn+WjNp0iS1HI/69etbrl27lvjajRs3LA0bNlSvdenSJcl22L+2XdWqVS0XLlyw+T6WLFliOX/+fIrl69evt4SGhloCAgIsZ8+etblvvJfUSO39LF26VC0PDg62rFixIslrP//8s3oNx9y7d6/N84bH+PHjk7y2atUqi4+Pj8XPz099LhlFsxGPtm3bWu7evZv4GvZTtmxZ9drgwYMz9P4vXrxoCQsLU699/PHHlvj4+MTXtmzZYsmRI4d67aeffrL5Gffu3TvDthNidBjWIcQBtNJghBUcoXDhwspV7+ub9CuIsMkPP/yg7sJnz54tzgT7tQ4/weszfvx45YGA1+Hs2bM2txs7dqwK2djiiSeeUKGM5DRo0ECFkFC99McffzjtPXz11Vfq74ABA5Tnw5q+ffuqUA+O+d1339ncHp6IF154IcmyZs2aSYsWLZQnDB4Pe8mSJYs6j/hr/fkiVAS+//575QlJD4T5kGQNLwxCe/hcNGrXrq2WgS+//NJuGwkxGwzrEKIjKDVGSOX06dMqt0PLe0AeBQQQwjE5cuTI9HEQjqlevXqK5chxqVGjhmzfvl3+/fdf6datW5LXIb6SVyAlB2EehCP27t2r7IU4AEeOHFF/kTTsDGJjY1X4AqRWNguBgnBSaiIDuTC2QD7NsmXL5Ny5c3bbhTJyW+INQglhOZwfnF+EoNJC613Tu3fvVN8bwk84r+fPn1d5NoR4KhQnhDiAlp+BfABHwHbohYJ8h7SIjIx0ijgpUaJEmq9h8LTlObFOUk3tbh9JtXfu3EnzPTgDDPKaByK194O8C5CayEB+ii3QOA9kxMNhz7nF+YPdqXmlrNFsTm1/8HTB83X9+nW1P4oT4skwrEOIA8D1DjCoIxxgL88//7wSJgh/rFixQi5duiTR0dHKc4KHFiqxt4IkM9g6lnWoIjlI3kWIBL1cPv/8c5U0e/v2bYmPj1f7+vHHH1Pdr14kD6O5CyOdA0LMAMUJIQ4Alz0GOpT8Lly40K5t4WVYsmSJ2h5/kTuB8ElAQEDi6xcvXnSqvagoSY2TJ08m5knYA3JiMOii3wvKiREaQbWOliuhhXWcBUIkKK3VSrltoS0vVKiQuAtnnVvN5tTeG/JR4DWxXpcQT4XihBAHQPiga9eu6v9vvvlm4qCRVhhHy73AIANvC0IJcNUnZ+rUqU6/0969e7d6JGffvn3K+wOhhDJYe9DeM/pxJAfhEZT12gL5NFoOiT0gSVgrlUXLdlv88ssv6q/WS8UdwPNlK7wH4YmQTkhISKKnLS20viToV5LWe0NfHGtx4uj5JMTIUJwQ4iBjxoyR0qVLqztnDJq28kcQqsGggqRT9OIA6O+BPBJ4XaZMmZJkffSsGDJkiNNthdh56aWXkjRcg0jCMryG/JciRYrYtU94SrTBFE3lrIUJqmlS8yhoXgR4VrTk2YwCIahVHq1atSrJaxAs8GLBA2XdW8XVoK8JzqN1wzUkrGq2vvjiixnqc9KvXz8lWCEW0evGWqDu2LFD9XYBb7/9ts3zibAaIZ4CE2IJcRAIDFSPoDMpKi1Q1YJkxqpVq6qSYOSRoIMq8jAw6GgJjH5+fqrjKhJJ0VV03LhxamJAVOygeqdHjx6qcgZdR50FGsWhmgbHgVdBa8IG7wfuxFEubC/oTIqSXQyceN94/3hvqD7CQA2BYKukF0mpKI1FN1dUC+H/GLzRhO2zzz5L85goXX7//ffVQI1wWKNGjdT+0FUVgzqOj7JeNIVzF/gMUSGEc4tzAHGGhmkIzyGnCM3YMgJE67Rp06RTp06qbBjCFaJWa8IGzwjOOUSMNfXr11fXFj6HmjVrqnMKgYYmbsmFDCFmgZ4TQjIBckVQtoouqxikMDjijn7OnDnqThaDE7qOwouAFu8ar7/+uixYsECVlyLc8+eff6rEUgiV1Nz6mRVS8MpASG3ZskUNpsgPefXVV9VyR/q1ICQFgQEvCf6Pc7BhwwZVWguhYKt0WQMhH5Qto5Jn5syZquvpjBkzMnTcjz76SB0LQkXrDAtPBQZ1iLvUusO6CggznAeIPohKdHFFQjMEKLrDppVUbCuXCecO5cQQtbiOkHgM0YPzo4V2rEFYB8eEAEUVD8KCOJ8o7ybErPigE5veRhBCiNnA3DnwimDG6bTmNCKE2A89J4QQQggxFBQnhBBCCDEUFCeEEEIIMRTMOSGEEEKIoaDnhBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oQQQgghhoLihBBCCCGGguKEEEIIIYaC4oS4hcmTJ4uPj4+cPHnS7m3XrFmjtsXf9MD+sS6O504biXHZsmWLNGzYULJly6Y+3507d8qIESPU/12JPdctISQpFCfElEyfPl1Gjx6ty7G1gS21x8WLF5OsHxkZKSNHjpRq1apJ9uzZJUuWLFK5cmV599135fz584nrPfvss6nuMzg42GF74+Pj5YsvvpASJUqo/VStWlV+//33DG8fEREh/fv3lzx58qgB/pFHHpHt27enWO+NN96QmjVrSs6cOSVr1qxSoUIFda5u376dQiy88sorUqlSJbW/okWLyjPPPCOHDx8WZxMTEyOdOnWS69evy7fffitTpkyRYsWKiZHQRHFqj40bNyZZ//79++q91KtXT8LCwtRnWrZsWXVOrc+hvdepPUycOFF9vjh2mTJlZMyYMRneNioqSl37BQsWVN8FvI+//vorxXorVqyQvn37qu+Kn5+fFC9ePNV9XrhwQV2juMaxz1KlSsmgQYPk2rVrDr9Hoi/+Oh+feAk9e/aULl26SFBQkN3bNm3aVO7duyeBgYFJxMnevXvl9ddfT7IuBh6sGxAQIK7mhx9+UGIjOeHh4Yn/P378uDRv3lxOnz6tBkn8gOJ97N69W/3Az58/P8mAgvPz888/p9gnfpwdZejQofLZZ59Jv379pE6dOvLHH39It27d1ACFzyQ9YfPkk0/Krl275O2335bcuXPL999/Lw8//LBs27ZNDUzWoqNJkybSp08fNWjt2LFDHXflypXy77//iq9vwr3Q559/Lv/99586HxBKGCTHjh2rhA0GYgxGzuLYsWNy6tQpmTBhgjz//POJy99//30ZPHiwGIkPP/xQDa7JKV26dOL/r169Ki1btlTnvnXr1upzxDV46NAhmTFjhvz0008SHR1t93VqDz/++KO8+OKL0qFDByUA1q5dK6+++qrcvXtXiY70gAifM2eO+u7i+oE4a9WqlaxevVoaN26c5Ds+c+ZMdV1AyKQGxG+DBg3kzp07MmDAAClSpIi6XnFNYZ84V9q1R0yEhRAT8uSTT1qKFSvm1H1OmjTJgq/EiRMn0lxv+PDhar0rV66kuV5MTIylWrVqlqxZs1rWrl2b4vWbN29a3nvvvcTnvXv3tmTLls3iTM6ePWsJCAiwvPzyy4nL4uPjLU2aNLEULlzYEhsbm+b2M2fOVO919uzZicsuX75sCQ8Pt3Tt2jXd43/11Vdq+w0bNiQu+++//yxRUVFJ1jt8+LAlKCjI0r17d4sz+eeff1LY7y5Wr16tjo2/GbnutmzZkqHr3tfX1zJnzpwUr92/f9/y5ptv2n2d2sPdu3ctuXLlUnZYg88N1+7169fT3H7Tpk3Kpi+//DJx2b179yylSpWyNGjQIMm6586ds0RHR6f7fZ82bZra56JFi5IsHzZsmFq+fft2u98n0R/KSeIWbOVzwE2Lu79169ZJ3bp11d12yZIl5bfffkszdo+79sWLF6s7Ys1Frbl8beWcwEuBuzXsG8fInz+/PPfccy53+c6dO1fdwcFzYX1HqBEaGiqffPKJwx4BPNIDXhKENnBHqYHz89JLL8nZs2dlw4YNaW6PO9x8+fJJ+/btE5chvIMwDPYNF31aaJ8LQkMayP+w9oIB3EEjzHPgwAFxFvjMH3roIfV/eGnwvnHtAFs5J3iO0MiCBQuU9wZeLNi0bNmyJOvhusP5LFeunAoh5MqVS+3f1blKmzZtUtc9Qh3wWiQH9n711VcO7RuevYMHD6a7HjwR+N5YX0/g5ZdfVp4L2Jfe9QQvIDyIGvhO4j3hWjxz5kzicnhLMuIBRdgU4Dq1pkCBAuovPiNiPhjWIbpy9OhR6dixo/px6t27t/zyyy9qUKlVq5YaGGyBwf7mzZtqcEXsHdhyW2sgno3wCsINECb79u1T7m/8RRjB0cRI5DEkx9/fP9FdvnDhwsSQlj3AdZ8cDOYQMxqPPvqo+pvegIjQCvI6kB9gDcSg9rot4WS9Pdzqyd3i2B7nECGpKlWqJC6PjY1VQgShBYTdED4JCQlJPF5qWCwWuXTpUqqfuSO88MILUqhQIfn0009V2AEhreQDWHIglOfNm6cGX9j9v//9TwkBDN4QIVr4av369SokVrhwYfUZIHQC4bN//36Vb+MIuKaTf/a4NrXjOno9pXedgl69esk///yjPoe0wPUAateunWQ5vq+4RvB6jx490twe+THW1zLQrg8kKyMsY2/YF8d+7bXX5Ouvv1afCW5IIPyffvppKV++vF37I8aA4oToCmLlyEdArgLAHTl+nCZNmpTqXeBjjz2mBp0bN26k+UOogYHmzTffTLKsfv360rVrVzUYace2F9w521qm3YHCC4CERXt+bHH3Cc9Eclq0aJHiDj4jIFEQA3JyAabdVVon5Ka2PX78k2O9vbU42bp1q4r/W58PDKpIkk2LadOmyblz51TehbOAHfDsQJzgM4YITg98ZhAYSKgESP5FIjMSiOFVAcjBSb6vNm3aqOPBW2aveNBAbpItbwgSYDXbgPX5dsZ1ag+4HuD5yJs3bwrxDBGVketJu3YcuR5tUbFiRSWU33rrrSTXHm52bOVvEXNAcUJ0BT8s1uIAAzN+OOHpcBbWbl380COBDuIEoOrEUXGCgSj5HSC8FNbuZtx92wNc3H/++WeK5UhEtSajIQQkB9tKQtaqf/C6M7fH5wlPFUQWvAtIhk1erZMcDJIIC2BgwYCiJxAImjABSNjFZ2x9PVpfTwiZ4XNG0io8EbieHBUn48aNU16F1BKhtfCFvddUetcpyGi5c/LE9OTXhLOvp4yCmxV4X5BYi6R4JOnC64XvjaOhLqIvFCdEV1BGmpwcOXIor4izgFsbpbyoZrh8+XIKV7qjwKOQXDRYk3xQywgYjGzdQTsKBlJbeSHa3Xh68Xh7t8d71uxv27atqrjAXwza8EAkB5U68ETAw6TlI6QFBq/knxlCde68HmHDqFGjlHcP3h7rUEhmricMrsnDJdZoAuPWrVt2Vdqkd53aAz7v5NVA1teEs6+njIDKL+SuIUSrnT+Ec3C+8L1HfhlEMzEXTIglupLaYJRe7NseECpCKSnKH5FPgP4JWogEpbKuArFuDFbWSX7uBu5yCIDk5xPudZBWiaa2vbauI9tribQQhsnBuXniiSdUjgo+j/T2BVBaCpusH+6+HgcOHKjyGXBdzZo1S11P8BYhrOHq6wns2bNH9ALnOy4uLoXIh2BBoqyrr6fUSpsRukwu7J566in1ucGDR8wHxQkxJRlNYsUd76pVq1RPC9xFtWvXTuWsoHLH1SAPAUydOlX0onr16qr/RPIqGFR+aK+ntz28HskHXWyPxM/kYYjk4C4Z2yb3KOBOGecHCbWLFi3K8J0tcm8gBKwf7gYeHoSfkHyJ3BNcT0gqtq5I8uTrScstsgbP8Tln5HrCZ66FqOy9Hm2BRGoIpuQg5KYlaRPzQXFCTAli5hlxoWt3wsk9B+7oLouBC8mLuMu2VbIL9zwqj1xZSoyQCsox0ThNA+di/PjxKk6Psl7ru1fkf2g/6tp7wI8/PE4aqCiZPXu2Giy1/AEMzNbbaWgJidZ3tRhIOnfurM4J9mOdxJgeuPNG2Mj64W5wTSW/ntAh1dYA6UxwntCADecU5c7JgfcCSaGuLCVu1qyZSm5GdZI1eA6xihCd9XWCfUIcW19POE9IYLUWsAiRoVOsvZU6AAIZ12jyvBmtC3KNGjXs3ifRH+acEFOC0kW4+NGhEiWiKCXW7iytQdwZMXe0b8fgiQEZbvgTJ0445Q7aVgkz7qThZoYowKCOARQ2IAzQqFEjtRxlzMjHQD6Dda8T3OWldmcMr4+WyJjRUmKUVaIT55dffqneP84VBjYkDKJCxjqMMWTIEPn111/VudH6k2AwQfIwyrBRxaJ1iMUAA0+UBgYGlOtiffQswUCJY+D9Q5hYV1WhcgoVPPi8kA+U/P1mpAJLT5DfgDb4yJOBxwciC4m/WsmvoyxdutSmQICA1Dx96AH0+OOPq3AZzh+uA1wTR44cUaEzCMzkCaDpXaf2lBIjJ+Sjjz5SCczo7QJPFj5nfIa4jq2rstChFdcIeqNo/WUgQLAdrjWEhpBIjGsO1zE6JluDcmCtfBotB3Az8vHHH6vnyF/Svu+oooK4wXOE3JAQi/cCcYL3iGMSE6J3FzjiHdjqvoqOj8k7TYKHHnpIPdLqtHn79m1Lt27dVKdSvKZ1j8T+8RzHs+6S2q5dO7VuWFiYpVOnTpbz58+r9dBFMy0bbaF13kztkbwj6I0bN1S3yipVqqhuscHBwZbKlStbhgwZYrlw4UKSDrFp7Tf5uctoh9y4uDjLp59+qtYPDAy0VKpUyTJ16tQU62nHT/7+0fWzb9++qjMo7Mdnk7yb6dGjRy29evWylCxZ0pIlSxb1HnEcnCt8VtZg+7TepzPRrp3kHWK1z9AaPLfupKuB84ZzY/159unTx5I7d25L9uzZLS1atLAcPHgwxXr2dohN7WF9LWtdWtF5t06dOur4+EzLlCljGThwoPocHLlOtc8ko/z000+WcuXKqWOju+u3336rOg9box0/+ftHR9i33nrLkj9/ftUVGO9j2bJldp0X6/MMcP47duxoKVKkiOqIjM8Cx7hz506G3xMxFj74R2+BRAghhBCiwZwTQgghhBgKihNCCCGEGAqKE0IIIYQYCkOJE8yxgoxrNOJBHwtb5XLJQZUAJiZDSSMyv61noyWEEEKI+TCUOMF8HCgRwxwTGQElj6irx+RcmM0SJZPPP/+8LF++3OW2EkIIIcQ1GLZaB56T+fPnqzkSUuPdd9+VxYsXq6nZNTCNudYOmxBCCCHmw9RN2ND8KHmHSDQFggclNdCN0HriKbRcRiMoNFDKaEt0QgghhIhq3Idu10jH8PV1XjDG1OIEE5ppHQ418BzzNmDmUFszXGI2UevOloQQQgjJHJjgFB2pnYWpxYkjoG0yWp5roCUypknHidWmJCfEMFxeK7KmdfrrPbxIJG8Td1hEnA0/Y2JCb8natWvlv//+U9NTYNqKkJAQpx7D1OIkf/78asIna/AcIsOW1wSgqkebrMwabENxQgxH9pYiewuL3D33oHN3cnxEshYWKdlSxPf/58khJoKfMTGZMFmzZo1s27ZNFaRUrVpVLXd2WoShqnUcmaVz1apVSZZhCnV7ZjklxNBgMKr13YMnyb/8D57XGs1By8zwMyYmEiarV69WbT+Q79m4cWOXHctQ4uT27duqJBgPrVQY/8d03lpIBrNnarz44oty/Phxeeedd9RsnpgtddasWfLGG2/o9h4IcTpF2os0mSOStVDS5bibxnK8TjzjMw7Kk3Q5P2NiIGHy999/q3AOZnvGDOuuxFBhna1bt6qeJRpabkjv3r1VczVMB64JFVCiRAlVSgwx8t1336lknJ9//llV7BDiUWBwKtRW5Mpaka0DRQLCRJr/w7tpT/uMo66JbO4vUv8XkWwlRPI04WdMDCFMEKVAjgmEScOGDb23z4m7QGVPWFiYSoxlzgkxBdsGiZz7U+SpI3pbQpzN1ldFLq4QaX1Qb0sISSFMHn/88RRpE64aQw0V1iGEZIDwSiK3j4nE3tXbEuJsbuwUCa+utxWEJAqTlStXKmGCiIQ78zkpTggxG2GVE6o6Inl37VFY4hPESQ6KE2IMYYICk/Xr10vLli2lfv36bj0+xQkhZiOsYsLfiP+ftoF4AHdOisTeojghhhAmK1asUF3YIUzq1avndhsMlRBLCMkAASEi2YqL3KQ48SjgNQE5qultCfFyYbJ8+XLZtGmTPPHEE1K3bl1d7KA4IcSMhFUSublPbyuIM7mxSyQ4r0hwfr0tIV6KxUqYtGrVSurUqaObLQzrEGJGwiszrOOpybCcgJToJEyWLVumhAk6v+opTADFCSFmTYq9e1okJlJvS4izYDIs0VGYLF26VDZv3qyECebL0RuKE0LMGtYBEQzteARR1xPEJsUJ0UGYLFmyRLZs2SKtW7c2hDABFCeEmJHQ8iI+vsw78RQidiX8pTghOgiTrVu3Sps2baRWrVpiFJgQS4gZ8c8ikr00K3Y8KaTjFywSUkZvS4gXCZPFixer2YWfeuopqVGjhhgJihNCzAqTYj2rUiesiogvf5KJ64EwWbRokWzfvt2QwgQwrEOIqcuJKU48AibDEjcKkz///FMJk7Zt2xpSmACKE0LMXLFz/5LI/at6W0IyQ1y0SOR+ihPiFmGycOFC2bFjhzz99NNSvbpxrzmKE0LMHNYBTIo1NxAm8TEUJ8QtwmTXrl3Srl07qVbN2J2IKU4IMStInvQNYGjHU9rWh1fR2xLiocTHxycKE3hMqlatKkaH2VeEmBUIk5By9Jx4QjIsKq8wZxIhLhImu3fvVh6TKlXMIYIpTggxM6zYMT9MhiUuFCZ//PGH7NmzR9q3by+VKz8IBZsAhnUIMXtSLMI6FovelhBHwOdGcUJcJEwWLFhgSmECKE4IMXs5cfQNkXsX9LaEOIKaHymC4oS4RJjs3btXOnToYDphAihOCDEzrNjxjGTYHMaunCDmEibz589XwqRjx45SqdKDebhMBsUJIWYmWwkRvyys2DFzMmxQLpEshfS2hHiIMJk3b57s379fCZOKFSuKWWFCLCFmxtdPJKwik2LN7DkJry7i46O3JcTkxMXFKWFy8OBBJUwqVKggZoaeE0LMDtvYmxcmwxInC5NOnTqZXpgAihNCPKJiZ7+IJV5vS4g9REeI3DlBcUIyLUzmzp2bKEzKly8vngDFCSGekBQbe1vkzmm9LSH2ELE74S/FCcmEMJkzZ44cOnRInnnmGY8RJoDihBBP8JwAhnbMF9LxDRQJLae3JcTEwuTIkSPSuXNnKVfOs64jihNCzE7WwiIBoSwnNmOlDoQlpiEgxE5hMnv2bCVM4DEpW7as3iY5HVbrEGJ2UOmBpFhW7JgLJsMSB4iNjVXC5NixY8pjUqZMGfFE6DkhxJPa2BNzEB+T8HlRnBAHhUmXLl08VpgAihNCPKac+IBIfJzelpCMEHlQJD6a4oTYJUxmzZqVKExKly4tngzFCSGeUrETHyVy+5jelhB72taHV9XbEmIiYXLixAnp2rWrxwsTQHFCiCfAih3zJcNi6oHAML0tISYQJjNnzlTCBB6TUqVKiTdAcUKIJxCcVyQoN5NizQKTYUkGhcmMGTPk5MmTymPiLcIEUJwQ4kkVO/ScGB+LRSSC4oSkTUxMjBImp06dUsKkZMmS4k1QnBDiURU77HVieO6dE4m6RnFCMiRMunXr5nXCxJDiZNy4cVK8eHEJDg6WevXqyebNm9P8AD/88EPl6sL61apVk2XLlrnVXkIMlRQbeVgkLkpvS0hGkmEpTkgawuTMmTPSvXt3KVGihHgjhhInSPoZNGiQDB8+XLZv367ERosWLeTy5cs213///fflxx9/lDFjxsj+/fvlxRdflHbt2smOHTvcbjshhvCcWGJFbh3W2xKSXjJsQLhI1iJ6W0IMKEx+//13JUzgMcGNurfiY7EgAGoM4CmpU6eOjB07Vj2Pj4+XIkWKyMCBA2Xw4MEp1i9YsKAMHTpUXn755cRlHTp0kCxZssjUqVMzdMzIyEgJCwuTmzdvSmhoqBPfDSFuJvqGyJycIg1/FyneRW9rSGqs7SQSdVWk+Wq9LSEGIjo6WgmTc+fOKY9JsWLFxAy4agz1NdIHs23bNmnevHniMl9fX/V8w4YNNreJiopS4RxrIEzWrVuX6nGwDU6m9YMQjyAwh0iWgkyKNTqs1CEeIkxciWHEydWrV9VkRvny5UuyHM8vXrxocxuEfL755hs1+RG8LH/99ZfMmzdPLly4kOpxRo0apVSe9oBnhhCPgW3sjU3MLZHbRylOSBJhMn36dDl//rz06NGDwsRo4sQRvvvuOzW3QPny5SUwMFBeeeUV6dOnj/K4pMaQIUOU+0l7ILZHiMfACQCNTcTuhL8UJ+SBMJk2bZq6oYYwKVq0qN4mGQbDiJPcuXOLn5+fXLp0KclyPM+fP7/NbfLkySMLFiyQO3fuqJKrgwcPSvbs2dMsuwoKClJxMesHIR5VsXP7uEjsXb0tIamFdHwDREIr6G0J0RmkGECYIDIAYUIvvkHFCTwftWrVklWrViUuQ6gGzxs0aJDmtsg7KVSokOqmN3fuXGnbtq0bLCbEqG3sLSKRB/S2hKRWqRNaUcQvUG9LiAGECW6+e/bsSWFiZHECUEY8YcIE+fXXX+XAgQPy0ksvKa8IQjWgV69eKiyjsWnTJpVjcvz4cVm7dq20bNlSCZp33nlHx3dBiI6EVUz4y9COMWEyrNejCRO0yIDHpHDhwnqbZEj8xUB07txZrly5IsOGDVOururVq6umalqS7OnTp5Pkk9y/f1/1OoE4QTinVatWMmXKFAkPD9fxXRCiIwHZRbIVZ1KsEYmPFbm5R6R4N70tIToKE7S5wDgHjwk8/sQEfU70gH1OiMexpo2IJU7kkSV6W0KsublfZHElkUdXi+R7WG9riJvBzTQ8Jp4mTCI9vc8JIcSJSbH0nBi4bX01vS0hOggTeEzQMgPpCZ4iTFwJxQkhnpgUe/eMSPRNvS0hyZNhsxZNaJZHvE6YXLt2TXlM0NmcpA/FCSGeRnil/w8jEOPAZFiv4969eyoPEsIEHhMKk4xDcUKIpxFaXsTHl6EdI4HUvhs7KE68UJjcuHFDCZMCBQrobZKpMFS1DiHECfgFi4SUYTmxkbh/USTqCsWJlwmTiIgIJUxSayRKUofihBBPbWNPz4kBk2EpTrxBmPz222+qeoXCxHEY1iHEYycA3Ke3FcQ6GTYgNKEHDfFY7t69q4QJymt79+5NYZIJKE4I8dRy4vuXRO5f0dsSonlOwquJ+PjobQlxgzCBx0RrHkocg+KEEI+dYwcVO/SeGIIIVup4gzC5deuW8phQmGQeihNCPJGQ0gmz3zIpVn9i74hEHqY48VAw/xvmg7t9+7YSJnnz5tXbJI+ACbGEeCIQJigppudEfyL2JMwUTXHikcIEHhP8hTDJkyeP3iZ5DBQnhHh0Uiw9J4bIN/Hx+/8Zo4lHeUwQ0qEwcT4M6xDiyUmxCOt499yexqjUCa2Q0H+GeAQI4UCYoGz42WefpTBxARQnhHhyr5OYCJF7F/S2xLth23qPFSbwmOTOnVtvkzwSihNCPL5ih6Ed3YiPE4nYTXHiYcIkKipKeUwoTFwHxQkhnkr2EiJ+WVixoye3j4rE3aU48QBQJqwJE3hMcuXKpbdJHg0TYgnxVDD5H5Iw6TnRv209GrAR0wuT6Oho5THJmTOn3iZ5PPScEOLJsI29/smwWQqJBNP9b3ZhEhMTQ2HiRihOCPH0ih2IE0u83pZ4J0yGNTVoRT958mQlTBDKoTBxHxQnhHi65wQdSu+c0tsS74Rt600tTOAxiYuLo8dEByhOCPH0cmLApFj3c+9SQhk3xYnpuHnzpvKYQJjAY5IjRw69TfI6KE4I8WSyFhYJCGXeiR5E7Er4S3FiOmECj0l8fLzymFCY6AOrdQjxZHx82MZez3wT/+wi2UvqbQnJIBEREUqYAAiT8PBwvU3yWug5IcRb2tgT91fqhFdNKOkmphImCOVQmOgLvzWEeEPeSeRBkfhYvS3xLpgMazph4uPjQ4+JQaA4IcTTQVgnPkrk9jG9LfEeYu8lCEKKE8Nz48YNlfwKYQKPSVhYmN4mEYoTQrwkrAMY2nEfyPFBbxmKE8MLE3hMfH19lceEwsQ4UJwQ4ukE5xUJys2kWHcnw6rpAx4IQ2JYj4mfn58SJqGhoXqbRKxgtQ4h3gDb2Ls/GTaknIh/Fr0tITa4fv268pj4+/urUA6FifGg54QQb4AVO+6FybCGFibwmAQEBNBjYmAoTgjxFs/JrcMicVF6W+L5INcEnhOKE8Nx7do1JUwCAwOVxyQkJERvk0gqUJwQ4i3lxJY4kchDelvi+dw+LhJ7m+LEgMIEoZygoCAKExNAcUKINxD+YI4d5p24JxkWhFfT2xLygKtXryqPCYWJeWBCLCHeQGAOkSyFWLHjDhDSCc4vkiWf3paQB8IEHpMsWbJIr169JHv27HqbRDIAxQkh3gKTYt3nOWFIxxBcuXJFfvvtNwoTE2K4sM64ceOkePHiEhwcLPXq1ZPNmzenuf7o0aOlXLly6uIrUqSIvPHGG3L//n232UuIqfJOGNZxPazUMYwwgccka9asKpRDYWIuDCVOZs6cKYMGDZLhw4fL9u3bpVq1atKiRQu5fPmyzfWnT58ugwcPVusfOHBAJk6cqPbx3nvvud12QkxRsaOSNe/obYnncv+qyN2zFCcGESbZsmVTHhP8JebCUOLkm2++kX79+kmfPn2kYsWKMn78eKV6f/nlF5vrr1+/Xho1aiTdunVT3pbHH39cunbtmq63hRDvbWNvEbl5QG9LPJeIXQl/KU50AzezSH6Fp4TCxLwYRpxER0fLtm3bpHnz5onLMN8Bnm/YsMHmNg0bNlTbaGLk+PHjsmTJEmnVqlWqx4mKipLIyMgkD0K8grCKCX+ZFOvafBO/LCLZS+ttidcKE3hMUI1DYWJu/I2UUR0XFyf58iXNcMfzgwcP2twGHhNs17hxY7FYLBIbGysvvvhimmGdUaNGyciRI51uPyGGxz+bSLYSzDtxdaVOeFURXz+9LfE6Ll26pJJf0fG1Z8+eyutOzIthPCeOsGbNGvn000/l+++/Vzkq8+bNk8WLF8tHH32U6jZDhgyRmzdvJj7OnDnjVpsJ0RVW7LgWJsPqwsWLF5XHhMLEczCM5yR37txqdkioX2vwPH/+/Da3+eCDD9SF+Pzzz6vnVapUkTt37kj//v1l6NChKiyUHDThwYMQr02KPTlFbys8k7j7Cfk8ZQbobYnXCRN4TMLDw9V4gMpNYn4M4znBXAe1atWSVatWJS6Lj49Xzxs0aGBzm7t376YQIBA4AGEeQoiNcmJUk0RH6G2J53Fzv4gllp4TN0Jh4rkYxnMCUEaMevTatWtL3bp1VQ8TeEJQvQOQ4FSoUCGVNwLatGmjKnxq1KiheqIcPXpUeVOwXBMphJDkFTsPBtI8DfW2xgPb1vuIhFfR2xKv4MKFC0qY5MyZU3r06EFh4mEYSpx07txZ1acPGzZMKeLq1avLsmXLEpNkT58+ncRT8v7774uPj4/6e+7cOcmTJ48SJp988omO74IQAxNaTsTHL6Fih+LE+cmwIWUSEo+J24QJPCZo2kk8Cx+Ll8c/UEocFhamkmORTEWIx7Oogkj+x0Rq/09vSzyLlQ8lzKnTeKbelng058+flylTpkiuXLmUx4TCxDPHUMPknBBC3ATb2Dsf3ONxTh2XAw85hYl3QHFCiDdW7LARm3O5c1IkJpLixA3CBJWdDOV4PhQnhHhjUuz9ywkP4sRkWLatdxVnz55VwiRv3rzKY8J2EJ4PxQkh3ug5AQztODcZNihPQs4JcbowmTp1qhIm3bt3pzDxEihOCPE2QkqL+AaKRFCcOL0zrI+P3pZ4FOjgDY8JKjYpTLwLihNCvA1ff5HQ8sw7cSZMhnWJMIHHBB3CKUy8D0P1OSGEuAkmxTqP6Bsid05RnDgR9LSaNm2aFChQQE3wig7ixLug54QQbyS8UsIEgN7d5sh5+SaA4sRpwgQek4IFC1KYeDEUJ4R4q+ck5qbIvfN6W+IZIR3fIJGQsnpbYnpOnTqlhAmmKenatSuFiRdDcUKIN8+xA+8JyRwRuxLm00EuD3GYkydPqlBO4cKF6TEhFCeEeCXZiov4ZWXeiTNgMqxThMn06dOVMIHHJCAgQG+TiM5QnBDijfj4ioRVZK+TzBIXnXAOKU4c5sSJE0qYFClShMKEJEJxQog3h3YY1skckQdE4mMoTpwgTLp06UJhQhKhOCHEq8uJ94lY4vW2xPxt68Or6m2J6Th+/LgSJsWKFaMwISmgOCHEm8VJ3N2ESeuI42XE2UuJBITobYnphMnvv/8uxYsXpzAhNqE4IcSbe50AtrHPfNt6kmGOHTuWKEw6d+4s/v6sciIpoTghxFvJUkgkIIwVO46CBnas1LGLo0ePKmFSokQJChOSJrwyCPFWMEkdk2Id5+6ZhNb1FCcZFiYzZsyQUqVKSadOnShMSJrQc0KINxNWiZ6TzCbDUpyky5EjRyhMiF1QnBDi7UmxkQdF4mP1tsScybCBORPCYyRVDh8+LDNnzpTSpUvLM888Q2FCMgTFCSHeDMI68dEit47qbYl5k2ERHiOpCpNZs2ZJmTJllMfEz89Pb5OISciUhL1165aaqOnGjRtisTG7adOmTTOze0KIOzwnAKGdsPJ6W2O+sE7hp/W2wrAcOnRICZOyZctKx44dKUyI68XJtWvX5JVXXpG5c+dKXFxcitchVHx8fGy+RggxEMF5RILyPGhj31Fva8xD9E2R28eZb5IKBw8elNmzZ0u5cuWkQ4cOFCbEPeKkX79+8ueff8qrr74qTZo0kRw5cjiyG0KIEWDFjv1E7E74S3GSqjApX768tG/fnsKEuE+crFixQt544w354osvHDsqIcRYoZ2Lf+lthflCOr6BIqEMhVlz4MABmTNnDoUJ0SchNmvWrKq7HyHEQzwnt46IxEXpbYl5iNiVUIbty7brGvv371fCpEKFCgzlEH3ESY8ePWT+/PmZPzohRH8wyFriRCIP6W2JeWBnWJvCpGLFispj4uvLQlCiQ1gHmdf//POPtGzZUvr376+mu7alkmvWrJlJ8wghbhEnWsVODs6umy7xMQk5OiV66W2JIdi3b58qjqhUqZK0a9eOwoToJ04aN26c+P+//koZq2a1DiEmIjBcJGthUyfFxsfFy+m1p+XWhVsSUiBEijYpKr5+Lhok4WGKj6LnRET27t0r8+bNk8qVK8vTTz9NYUL0FSeTJk1yngWEEP0xcRv7A/MOyLLXlknk2cjEZaGFQ6Xldy2lQvsKrmtbH15NvBlNmFSpUkXatm1LYUL0Fye9e/d2rhWEEP0rds7ON6UwmdVxlkiyHpCR5yLV8mfmPON8gYJk2GzFRQLDxFvZs2ePyjukMCGuglcUISShYgdNxWLviJlCOfCYJBcmigfLlr2+TK3nVLw8GVYTJlWrVqUwIfp6Tp577jmVQ/LTTz+pxFc8Tw+sP3HiRGfYSAhxWxv7/SK56ogZQI6JdSgnBRaRyDORar3iDzup9QGm6YA4KfuKeCO7d++WBQsWSLVq1aRNmzYUJkRfcfL333+rizA+Pl6JEzyH+EiL9F4nhBiIsAehD7SxN4k4QfKrM9fLEPfOi0Rd9UrPya5du5QwqV69ujz11FP8jSf6i5OTJ0+m+ZwQYnL8s4lkL2mqih1U5ThzPbuSYb1MnOzcuVP++OMPqVGjhvKYUJgQV2NIn9y4ceNUB9rg4GCpV6+ebN68OdV1H374YfVFSf548skn3WozIR4R2jFRxQ7KhVGVI6mNkz4ioUVC1XpOTYYNQOm1E/dpEmGCvlUUJsTQ1TrWpWRLlixJ9KRAUDzxxBMqg9tRZs6cKYMGDZLx48crYTJ69Ghp0aKFmn47b968KdZHKVt0dHSSGZMRD+3UqZPDNhDitUmxx38Vs4A+JigXVtU6yXkwfrYc3dK5/U5UMmw1xK3FG9ixY4csXLhQCZPWrVtTmBC34WNBxzQ7iYqKkhdeeEGmTJmiGq5pSVHIScHF2717d/n5558lMDDQboMgSOrUqSNjx45N3Cc60A4cOFAGDx6c7vYQM8OGDZMLFy5ItmzZ0l0/MjJSwsLC5ObNmxIaGmq3vYR4DCeni6zvLtLxRkJjNhOVE8/vNV9i7sQkLoPHBMLE6WXEf5YVKdhKpNZo8XS2b9+uZp+vVauW8kRTmBB3jqEOeU7effdd+e2332TAgAFKNJQqVUpduEePHpX//e9/8sMPP0jOnDmVULAHeEC2bdsmQ4YMSVwG4dO8eXPZsGFDhvaBCqEuXbqkKkwgrPCwPrGEEOuKnX0ieRqJWYAA2fS/Teo3qGb/mq7rEBtzS+TWUa/IN8Hv8KJFi6R27drSqlUrChPidhz69k6dOlV69uypvBvlypUTf39/VcWD/yNfBJ4TrGMvV69eVS3v8+XLl2Q5nl+8eDHd7ZGbglDT888/n+o6o0aNUipPe8ArQwiBu6GciI+fqZJiNa4dvqYESZWuVVTZsEta10fsSahP9nBxogkTeLApTIheOPQNjomJkfr166f6esOGDSU2NlbcDbwmyHepW7duquvAKwP3k/Y4c+aMW20kxLD4BYmElDFVUiyIioyS2xduS65yuVx7IOSb+PiLhLqgJb5B2Lp1a6IwQf4ghQkxlThBgury5ctTfX3ZsmXy+OOP273f3LlzKw/MpUuXkizH8/z586e57Z07d2TGjBnSt2/fNNcLCgpScTHrByHEumJnn5iJq4euqr+5y+V27YFQqRNWMUHEeSBbtmyRxYsXq5s7ChNiCnFy/fr1JI+PPvpITpw4Ie3bt5dVq1bJqVOn1GPlypVqymz8H+vYCxJokXyFfWogIRbPGzRokOa2s2fPVrkkPXr0sPu4hBCrih2ThXWuHbqm/rrFc+KhIR2ExFF5iYKEli1bUpgQ3fHPqEcj+cWKKh3MsYD69+TLQaVKlRwK7aCMGBMLIhELCh5JtfCK9OnTR73eq1cvKVSokModSR7SwZTduXK5+AeKEE/3nERdEbl/WSQ4Zem+UT0nIQVDJCjEhR6N+FiRiN0ixbqIJwqTpUuXqlA9PN4UJsQ04gSlue66YDt37ixXrlxRx0QSLFolI0ykJcmePn06xXwO6IGybt06WbFihVtsJMRjCauU8BehHZOIE3hOXO41uXVEJO6+x3lONm3apH5fKUyIR/Q58STY54SQZB6CWdlEanwlUm6gmIHx1cZL4YaFpfUPrV13kJO/i6zvJtLhmkhQTvEENm7cqHIHETJ/7LHHKEyIocZQQ7avJ4TohO+DahSTVOxY4i1y7cg19yTDZi3iMcIEfaMgTFBZSWFCPK59PSHEAzFRUuzNMzcl9l4sk2HtFCYIgTdq1EgeffRRChNiSOg5IYSkzDtBzokJIr5XDz4oIy7vYs+Jh4iT9evXK2HSuHFjChNiaChOCCEpK3ZiborcOydmSIb1C/KTsKJhrjvIvYsi9y+ZXpz8999/8tdff0mTJk2kWbNmFCbE0FCcEEJShnWACUI7KCPOVSaXa9rVW3tNgInFCaoZ0YcKwuSRRx6hMCGGh+KEEJKUbMVE/LKaIinWLWXEECf+ISLZiosZWbt2rWpk2bRpUwoT4h0Jsfv375fjx4/LjRs3EpuvWYOGaYQQk+Hj+/95JyYQJ1V7VXV9pU6OagnnxYTC5O+//5aHHnpIHn74Yb3NIcS14uTYsWOqTTw6C6bWJgXqnOKEEDNX7GAWXuMSfSdaIs9Gur6MGJ6T/M3FbPz777+yevVqChPiPeLkhRdeUK3r0VoeMcwcOXI43zJCiL5JsadmopGIYT0G1w67YU6d2DsikYdEKrwlZuKff/6RNWvWKFECcUKIV4gTZH2/9957MnCgOTpIEkLsBGGduLsid06KZC8phi4jdqXnRCUFW0yVDAtRAnGC/BLkmRBiRhy6JcJEgGhXSwjxUExQsYN8k2z5sklweLBrQzo+fv8/55CBQYgdYRwIE5QKU5gQrxMnL774okydOlXi4uKcbxEhRH+yFBQJCDd0xQ7EiVva1oeWF/FzoQBykjCBxwR5JmiuhnA7IR4f1pk3b16S5+XKlVPCpFq1avLcc89JkSJFxM/PL8V27du3d56lhBD3gXJTg7exR4+TgrULird3htU8JqjMad68uWpLT4hXiJOOHTuq6hutMsf6/2+9ZTtRDOvQs0KIiUEo4+oGMSL4/UFCbJXuVVx3kPg4kYjdIkWMe5OF84BSYTRZozAhXidOoMoJIV5YsXN8kkh8jIhvgBiJW+duScydGNeGdW4fS6jWMajnBMIEzdVQoICZhTHDMCFeJU5YikaIF4KwTny0yK2jImEVxIiVOi4tI9ba1odXEyMLk8cff1waNGigt0mE6J8Qe/36ddm9e3eqr6MHCrrGEkJMjFahYsCkWOSb+Ab4So4SOVybDIvE4OA8YjRhgnlyIExatGhBYUI8EofEyRtvvCH9+/dPs0lbarkohBCTgEE5OK9IxD5DVurkLJ1TfP1dPOGfwUI6ECaYWXj9+vXSsmVLqV+/vt4mEeISHPpmIwHrqaeeSvX1Nm3aKGVPCPGAvBMDek7cUkZsMHECYbJixQrZsGGDEib16tXT2yRCjCVOrly5ohqxpUauXLnk8uXLmbGLEGIEDCpOENZxab7J/csi984bRpxAmCxfvlw2btwoTzzxBIUJ8XgcEicFChSQHTt2pPr6tm3bJE8eY8VpCSEOEF4pISE27r4YhZh7MXLz9E0XJ8PuSvgbXt0wwmTTpk3SqlUrqVu3rt4mEWJMcfL000/LxIkTZeHChSle++OPP2TSpEnSrl07Z9hHCNHbc2KJS5j8ziBcP3JdTXfj0rAOQjr+2URCSonewmTZsmVKmDz55JNSp04dXe0hxNAT/40YMULllECAoEts5coJ83Ds3btXdu3aJRUqVJCRI0c621ZCiF4VO+gUm6Oa95QRo1InvKquMzJDmCxdulS2bNmihEnt2rV1s4UQd+PQNw+T/iH2+f7770tMTIzMmTNHPfD/Dz74QKn88PBw51tLCHEvgWEiWYsYKu8E+SZZc2eVrLmyemwyLITJkiVLlDBp3bo1hQnxOhzynIBs2bIp7wg9JIR4gffk5j5DVeq41GsSe08k8qBIuVdFT2GydetWVflYs2ZNXewgRE/081kSQsyBwSYAdLk4gRBDno0OybAQJosXL1bCBO0aKEyIt+Kw5+T+/fsyd+5c2b59u9y8eVPi4+NTTPyHpFlCiAckxd75SiTmtkhAdl1NweCNsE7FThVdG9JBrglEmZvf26JFi9RvKoRJjRo13Hp8QkwvTk6dOiWPPPKInDx5UuWWQJzkzJlTIiIi1EzE6IGSPbu+P2KEECehDdI394vk1reM9fbF2xJ9K9r1ybAhZUX8XZjTYkOY/Pnnn6pFQ9u2baV6df1LmAkxXVjn7bffVoIESbGHDx9WX6yZM2fK7du35fPPP5csWbKounxCiAcQikn/fAyRd4KQDnB5GbEbk2Hx+4m2DBAmaNNAYUJIJtrXDxgwQDUD8vX1TfyCBQUFKeHy6KOPyuuvv+5sWwkhegAPQvaShqjYQRmxj5+P5Cjpogn/LPEJDdjcJE40YYIWDFprBkKIg+Lk7t27Urx4cfX/0NBQlV8CT4oGZslct26d86wkhOiLQZJikW+Ss1RO8Qv0c80Bbp8Qib3llmRY5OlpwgQek6pVq7r8mIR4tDgpWrSonD17Vv3f399fChUqpEI8Gvv375fg4GDnWUkI0ReDlBO7vFIHIR3gYs+JtTCBx4TChBAnJMQ2a9ZMtakfPny4ev7ss8/KqFGj5MaNG+pLN2XKFOnVq5cjuyaEGLVi5945kegbIoEuCqlkUJyUb1/etcmwwflEsuRz2SHwG4nfzz179kj79u0TO2wTQjIpTgYPHqw6F0ZFRak8k/fee0/Onz+vusT6+flJt27d5JtvvnFk14QQI1fsROwTydtYFxNio2Il4mSEqZNhIUwWLFigpvqgMCHEyeIEYR08NBDC+fnnn9WDEOKBhJQT8fFPSIrVSZxcP3pdLPEW14d1indzuTDp0KGDVKr0YN4iQohrOsQiGRb9TZzBuHHjVLItBE+9evVk8+bNaa6P3iovv/yyFChQQHlxypYtq1o/E0KciF+gSEgZXfNOXF5GHHVN5O4ZlyTDQpjMnz9fCZOOHTtSmBDiKnGC9sotW7aUrFmzSq5cueSff/5Ry69evaqaCK1Zs8bufaJXyqBBg1QuC7okoqyuRYsWcvnyZZvrR0dHy2OPPaaawSGkdOjQIZkwYYJK0CWEeFbFDsqIg3MES9Y8LmqOhhJi4OSwDoTJvHnzVKEAhEnFii7sbkuIN4uT9evXS+PGjeXIkSPSo0ePJK3r0R0WnpQff/zR7v0iT6Vfv37Sp08f9QUeP368Ej+//PKLzfWx/Pr168pV2qhRI+Vxeeihh9grgBBXJcXe3IPmHLocHp4TeE3QusBlIR2/LAkeIicBjzKm+Thw4ACFCSGuFidIgK1QoYK6E/j0009TvI7W9ps2bbJrn/CCbNu2TZo3b/7/xvn6qucbNmywuQ1K8dBTBWGdfPnyqeQy2OOsEBMhJJnnBKGP+7Y9me7oceLytvXhVUR8ndNDBb9D8JgcPHhQOnXqpH4zCSEuFCeo1IF3Azketu5iEFa5ePGiXftEOAhfZogMa/A8tX0dP35chXOwHfJMPvjgA/n666/l448/TvU4qDCKjIxM8iCEZLDXCdAh7wSdVN3S48RJIR3NY6IJk/LlXVj+TIgH4pA4CQgISDELsTXnzp1zy8R/sCFv3rzy008/Sa1ataRz584ydOhQFQ5KDfRjCQsLS3wUKVLE5XYS4hFkLyXiG6RLG/u7V+7K/Yj7rkuGjYtKmNjQCeIEwkTLgXvmmWcoTAhxlzipX7+++vLZ4s6dOzJp0iSV+2EPyFVBj5RLly4lWY7n+fPnt7kNKnRQnYPtNOA6hacFYSJbDBkyROXEaI8zZ87YZSchXouvv0hYBV2SYhHSAS7znECYWGIzXamjCRPk4+FmqVy5ck4zkRBvwiFxMnLkSFWt8+STT8rSpUvVMrRhRp8TeDCuXLmiQiz2EBgYqLZdtWpVEs8IniOvxBZIgj169GgSLw5mSYZowf5sgVAU5gOyfhBCjN3GXk345+sjOUvndGHbep+EnJNMCJPZs2crYQKPCW6cCCFuFCfoP4IcDwgDrU39m2++Kf3790/M/3BkrgiUEaMU+Ndff1XZ7S+99JLyxCC/BeBY8Hxo4HVU67z22mtKlCxevFglxCJBlhDiqoqdvW6v2EG+SXiJcPEPcqhvZMaSYUNKiwQ4Fo6OjY2VWbNmqd9EeEwoTAjJHA5/0zG/DmKqO3fuVHcK8F6UKlVKeT8cLfXDlxpel2HDhqnQTPXq1WXZsmWJSbKnT59WFTwayBdZvny5vPHGG0oMIREXQuXdd9919G0RQtKr2ImJFLl7ViRbEbeXERsxGRbCBB6TY8eOSZcuXaR06dJON48Qb8PHgjR4LwbVOkiMRf4JQzyEpMPtkyILS4g8vESk4BNuO+yYsmOkbOuy0uKbFs7fOX4C5+QQqfiOSKX3HPKYoHKQwoR4I5EuGkOd0r6eEOIlZCsq4p/NrXkncdFxcuP4Ddclw945JRJz0+5kWE2YnDhxQrp27UphQogTcVEAlxDikfj4JiTFurFiB8LEEmdxXVhHJcPa17YewgTTbWDqDHhMENImhDgPihNCiP1JsUgg9ZQyYryXoNwiWQpkWJjMmDFDTp06pTwmJUuWdI1dhHgxDOsQQuxPilV9QVJvxOjsMuLAkEDJnj+7a5NhM5DIHxMTQ2FCiBugOCGE2AfCOnH3RG6fcF+lTnkXT/iXgZCOtTDp1q0bhQkhRg7r3L59W27cuKHmvkhO0aJFM7t7QogRwzoA/U5CSpm7jDg6QuTOyXSTYTVhgo7S3bt3VzOgE0IMJk7u37+vusROnDhRrl27lup6nB2YEA8EuRmBORKSYgu3dUvOSeknXFQJc+NB7kwanhMIk99//13Onj2rPCYUJoQYVJwMGDBAdXF9+umnpUmTJpIjRw7nW0YIMSYIr7ipjf3da3fl3rV7rkuGRUgHkxmG2p4DB3N0QZhgMlN4TIoVK+YaOwghmRcn8+bNk+eff15+/PFHRzYnhHhCaOfqf24J6QCXhXVQqYMEX0xqmAwKE0JMlhCLxLSaNWs63xpCiDnAgB55UCQ+xvVlxD4iOcu4cMI/GyEdCJPp06fL+fPnpUePHhQmhJhBnLRt21ZWrlzpfGsIIebxnECY3Dri8jLisKJhEpAlwPk7j4tOCE0lS4bVhMmFCxeUMGFiPyEmEScffPCBmksCsxBv27ZNTdaH2YGTPwghHgpyToCL8060MmKXoDw/0Uk8J1FRUTJt2rREYYLJRQkhJsk5KVOmjPq7Y8cOVbGTGqzWIcRDCc4tEpwvoWKnaCeXipOSj5d0cdv6qkmEyeXLl6Vnz55SuHBh1xyXEOIacTJs2DDXNUQihJgntINeJy4iPjZerh+7LnXL1XVdMmz2kiIBoUmECTwmFCaEmFCcjBgxwvmWEELMlxR7fqnLdn/jxA2Jj4l3bRlxjupKmEydOlWFp+ExKVSokGuORwjJMGxfTwhxPO/k9lGRuPvmKyNGR+sbO+V+tmoUJoSY1XPy4YcfqjDO0KFDxdfXVz1PD6yPxFlCiAeHdTD5HxJLMzA3jSNlxAHZAiSkUIjT9y13z8r9e3dk6sYscu3WVenVq5cULFjQ+cchhDiEj8XWpDjJgCCB2Lh3754EBgaq5+nu2MfHFAmxkZGREhYWJjdv3pTQ0FC9zSHEPMREiswOE2kwRaRED6fvfmG/hXJh2wV5YfsLTt/3/WPzZersJXLNp6T07NmbwoQQg42hGfKcxMfHp/mcEOKFBISKZC3isnJiV5URY26wKYt3y/XY3NLrud5SgMKEEMPBnBNCSOZCOygndpE4cXYyLLy/U6ZMkRu3YqVXlX0UJoQYFIoTQkjmKnZcUE58P+K+3Ll8x6nJsInC5MYN6VVqmRQo7KKZjgkhmYbihBCSOc/JnZMiMbedP6eOiNM8JxAmv/32m0REREivbh0kf/xWlyTxEkKcA8UJIcRxwrU29vtdUkacq2zmxcndu3eVMEHiXu/evSV/4IWEFyhOCDEsFCeEEMcJrYDaPKeHduA5CS0cKoHZAp0mTFAunC9fvoTma76BIqHlnWYvIcQAHWIJIUThn1UkeymnJ8VeO5j5Sh1NmNy6dUt5TPLmzfv/bevDKor4ZU74EEIM5jn5/PPP5dy5c863hhBiPlyQFAvPSWbyTe7cuSO//vqr3L59O6kwsWpbTwjxMHGCTrHFihWTZs2ayaRJk9SdCSHEi9vYO7HXSXxcvFw/et1hcQJhAo8J/qYQJvGxIhF7RMIpTgjxOHFy6tQpGTVqlFy/fl369u0r+fPnly5dusjixYtN0RWWEOLkip1750WirjtldzdP3ZS4qDiHyog1j4kmTPLkyZN0hchDIvFR9JwQ4oniBJNjvf3227Jz507ZvXu3vPrqq7Jx40Zp06aNFChQQAYOHCibNm1yvrWEEGOGdYCTvCeOlhEjhANhgrLhZ599NqUw0UI6IEc1p9hKCDFotU7lypWVF+XkyZPyzz//SJMmTeT777+Xhg0bStmyZeXjjz+Wy5cvO8daQojxCCkr4uPvNHGCMmL/LP4SViTMIWECj0nu3Kl4XZAMm62YSGC4U2wlhBi4lBhzVcyYMUO++OIL+fPPP8XPz0+eeOIJJVw++ugjKVWqlMyfP98ZhyKEGA1UvYSWdVrFztWDVyVXmVzi4+tjlzCJiopSHpNUhQlgMiwhni1OMJnxihUr1F0Kegd069ZNzp8/rwTK2bNnZdGiRTJv3jzlUalVq5a8+eabzrWcEGKsvBMnVezYM+EfkvE1YYLfoly50ggFYQJ2iBMmwxLimX1O3njjDZk5c6ZcunRJ5Zi8+OKLqsFRpUoPukVagdeff/559TohxIPFyaVVCQLAJ2Mej7RyToo2KZphYRIdHZ2+MAH3LohEXaHnhBBPFScTJkyQdu3aKcHRvHlz8Unnx6hx48aq5JgQ4sFt7KOuidy/LJIln8O7iYqMktsXbqebDKsJk5iYGBXKyZkzZ/o7T0yGpTghxCPFCTwm2bJly/D6xYsXVw9CiAd7TgBCO5kQJ9cOJ8ypk1YZMVrRQ5jExsYqj0mGhAmI2CkSEJaQEEsI8byckypVqsjChQtTfR35JiVLlnTYqHHjxikxExwcLPXq1ZPNmzenuu7kyZOV58b6ge0IIW4ELex9gzKdFJtYRpzKhH+aMEE/pQx7TOLjRC6tETn7Z4IwscRnykZCiEHFCZJckSGfGngNjdocAbksgwYNkuHDh8v27dulWrVq0qJFizTLkUNDQ+XChQuJD0ePTQhxEF+/hPlqMpkUi2TY7AWyS1BoUIrXbt68qW5GIEzgMcmRI0f6OzwzT2RhcZFVj4hc2ygSsTvhOZYTQjyvWietPJMtW7ZIeLhjfQS++eYb6devn/Tp00cqVqwo48ePl6xZs8ovv/ySpi3oUqs91MyjhBDTtbFHGbGtkA6ECTwm8fHxymOSYWGytqPI3bNJl989l7CcAoUQ84uT7777ToVq8IAYeP311xOfWz+QMT969Ghp1aqV3cYg637btm0qyTbRQF9f9XzDhg1pemow10+RIkWkbdu2sm+f8+b5IITY0SkWYR1U7GTCc5KrfNKQTkREhPKYoH0BhEmGbnwQytn2GuqHbbz4YNm21xPWI4SYNyEWk2dppcII66CFPR7WQLQgURZ9TQYMGGC3MVevXlUu2+SeDzw/ePCgzW3KlSunvCpVq1ZVd1dfffWV6k4LgVK4cOEU66MfAh7WMWxCiJOSYmNvidw9I5It/VLg5JP9nfrnlFw5cEWVEeO5r5+vEibwmACEcjLskb38T0qPSRIsCXZeWSuS72G7bCWEGEicdO3aVT3AI488Iu+//748+uijojcNGjRQDw0IkwoVKsiPP/6outMmB632R44c6WYrCfGSsA5AaMcOcXJg3gFZ9toyiTybcKOwZdwWOfTHIWnyZRP579J/6qYHwiQsLI129tE3RK5uErm6MSG35PLajB0cvU8IIZ5RSrx69WrnW4Lywdy5Vet7lCpbg+fIJckIAQEBUqNGDTl69KjN14cMGaISbq09JwgHEUIyCQSJf/aE0E7BJzIsTGZ1nJUi+hJ5J1IWb1os2fJmk34D+iUVJgjFRO4XubohQYzgb+QDz2pQLpFcDUSKdRE5nnqeWiJZCtj1FgkhBhIn//77r/rbtGnTJM/TQ1s/owQGBqqQ0KpVq+Tpp59Wy5AAh+evvPJKhvaBsNCePXtSzXkJCgpSD0KIk/HxfZAUm7GKHYRu4DFJkRaCXNfeWEHE51cfCXk1SuTc4v8XI9c2J4SPfPxEwquK5GsmUuk9kdwNEkqakawPAXNxRULyq828Ex+RrIVF8jRxylsnhOggTh5++GHlWsWMnxAQ2vPUQOIaXodQsBd4NeDCrV27ttStW1cl1965c0dV7wB0pUWuC8Iz4MMPP5T69etL6dKlVXz6yy+/VKXEaJlPCHEzKCe+vE7k5O8JXgkM/igztsHptacTQzlJhMmzIhKLJkYit2/dltOj6knxiidFgvMmCJDKQ0Vy1RfJVVvEP5VmkDhmre8SqnIgRJIIlAe/XbVGp2obIcQE4kQL40CYWD93BZ07d5YrV67IsGHD5OLFi1K9enVZtmxZYpLs6dOnVQWPxo0bN1TpMdZFeSE8L+vXr1dlyIQQN4LSXDxiboqs75awDN4JiIQi7VOsfuvCrTSFiTx4+VbewSJPPS6Srbh98/bgmE3mJFTtWCfHKptG27SJEGIMfCxwc3gxyDlBPBuVPmjmRghxAK2nSIoQygMxAZFQsLVIxK7EPJGTq0/Irx+0THg95wNhEi0iKM6x0i29V/eW4g9nYvoLhHhQlYPk13S8OYQQY4yhDiXEEkKIXT1F/kOlH3JBokR8A0Vy1pKijzeQ0O/9JTI6NiHHJPqBx0RrPu0jElo4NEMzFKcJhAjLhQkxFRkSJ88995zdO0bOycSJEx2xiRBiJuCVSLOnCARMtEiZl0VK9EyYFdgvSHWAbPT1Zlm6dalI1AOPiZUwAS1Ht1T9Tggh3kWGxMnff/+dZgKsLexdnxBiUjLaKyRPI5Hc9ZI0XVx7fq2E5gmV+F/ik8zXBY8JhEmF9hVcYTEhxBPECTrCEkJIpnqFWK0HYYLOr1myZJFeL/SSrG9nVdU7SJINKRCiQjn0mBDivTDnhBCSOZBgigqYDPYUQTXeb7/9liBMevWS7Nmzq+WZSnolhHgUvDUhhGQOraeIInk4N2lPEQgTeEww0zj6GWnChBBC7BYn6Cvi7++vZg3WnqPNfFoPrE8I8RK0niJZk04GqjwmWF6kfaIwweSg8JjgLyGE2CJDCgIN0ZDgqgkO7TkhhCQRKIXa2uwpcvnyZSVMQkJCpGfPnhQmhJA0YRM2NmEjxKVYCxN4TBDSIYR4BpFswkYIMRuYURzJr/jRgseEwoQQ4tKEWMSP33rrLTWHDX5w8MD/sQw/SIQQ7wbzXcFjQmFCCHFLWGffvn3y6KOPKndtvXr1pGzZsmr54cOHZdOmTZInTx5ZtWqVVK5cWYwOwzqEuEaYwGMSHh6uhAnKhgkhnkekkcI6L7/8ssTFxSkhUqdOnSSvbd68WVq1aiUDBw506ezFhBBjQmFCCNElrAMB8tprr6UQJqBu3brqNQgXQoh3ceHCBRXKyZEjB4UJIcS9npO8efNKcHBwqq/jNaxDCPEuYQKPSc6cOZUwSes3ghBCnO45ef311+WHH35Q7tvknD9/Xr2GdQgh3gG+9xAmuXLlojAhhLjHc/LNN9+kWIa206VLl5Z27dqpv+DIkSOyYMEC9dzL26cQ4jWcO3dOpk6dqoRJjx49KEwIIe6p1kG7ert37OOjkmaNDqt1CMmcMJkyZYqq0IMwCQoK0tskQoi3VOucOHHCaQckhHgGZ8+eVR4T5Jd1796dwoQQ4l5xUqxYMecdkRBieihMCCGuhO3rCSF2cebMGSVM8ufPL926daMwIYQYR5zs3r1bxowZI9u3b1expvj4+BQ5J8eOHXOGjYQQAwoTeEwCAwP1NokQ4oE4VEq8Zs0a1Wxt0aJFUrBgQTl+/LiULFlS/f/UqVOqkqdp06bOt5YQohunT59WwqRAgQIUJoQQ44mTYcOGKTFy6NAhmTRpklr23nvvybp162T9+vUqHv3MM88421ZCiI7CZNq0aeoGBKEcChNCiOHECUI5ffv2VWVDfn5+aplWNoyJAF944QX54IMPnGspIUQX4A2FxwTCpGvXrhQmhBBj5pz4+/tLSEiI+j8m9woICFAzFGvAq7J//37nWUkI0U2YwGNSuHBhJUzwXSeEEEN6TtABFt1gtcTX8uXLy/z58xNfX7x4sUqYI4SYl5MnT1KYEELMI05atWolv//+u8TGxqrngwYNknnz5kmZMmXUY+HChSq0QwgxJ2i8OH36dClSpAiFCSHEmO3rkxMTE6Na1mL2UXhOAGLSc+fOVTkorVu3lmeffVbMANvXE2JbmBQtWlS6dOlCYUIIcfsY6pA48SQoTgj5f9AWAF5RdIXu3LkzhQkhxLhz66QGKnS2bdumYtOgRIkSUrNmzcQKHkKI+YRJ8eLFlTBB4jshhOiBw78+kydPliFDhqgqHc35ghAPZif99NNP5bnnnnOmnYQQF4JuzjNmzKAwIYQYAod+gX788Ud56aWXpHr16jJixAgpW7asWo6mbHitX79+Eh0dLS+++KKz7SWEOJmjR48qYYIWAGieSGFCCNEbh3JO8COGLP6VK1emiEkjWbZZs2Zy7tw55SY2Osw5Id4MhQkhxIhjqEOlxBcvXlQ/ZLaS5bAMGf6XLl1yhn2EEBeBXkUQJqVKlaIwIYQYCod+jWrUqCGHDx9O9XW8hpAPIcS4wmTmzJmqoWKnTp2YxE4IMRQOeU7GjBkjs2bNku+++07u3buXuBz///bbb9VrY8eOddiocePGqcS84OBgNVfP5s2bM7Qd7gKRlPv00087fGxCPB3cPFCYEEJMn3NStWrVFMuuX78uFy5cUK5gTAgGzp8/r7rGYkr1XLlyya5du+w2CD+avXr1kvHjxythMnr0aJk9e7ZKts2bN2+q26GcuXHjxip2juZwCxYsyNDxmHNCvAl8j3DzgCT2jh07UpgQQszbhO3hhx9O7ARrD6tXr7Z7GwiSOnXqJHpe4uPjVfLtwIEDZfDgwan2W2natKkqX167dq1ERERQnBCSijApV66cdOjQgcKEEGLuJmxr1qwRd4DyYzR1Q/8UDV9fX2nevLls2LAh1e0+/PBD5VXp27evEidpERUVpR7WJ5YQT+fgwYPKA0lhQgjx2JwTV3H16lXlBcmXL1+S5XiOCiFbrFu3TiZOnCgTJkzI0DFGjRqlVJ72gFeGEE/mwIEDSphg9nAKE0KIR4sTiIhff/1VlSAiFIMH/v/bb7+p19zBrVu3pGfPnkqY5M6dO0PbwCsD95P2OHPmjMvtJERPYTJnzhypUKEChQkhxLNLiTGot2jRQrZs2SIhISEqCRX89ddfambiH374QZYvX253/AkCAz+eyXuk4Hn+/PltttxGImybNm0SlyFHRb0xf38VY0cPB2uCgoLUgxBPZ//+/UqYVKpUSdq1a6dCpIQQYgYc+rUaOnSoyg1BSfGVK1dk+/bt6oF5dpDIunXrVrWOvQQGBkqtWrVk1apVScQGnjdo0CDF+nBT79mzR3bu3Jn4eOqpp+SRRx5R/2fIhngr+/btozAhhHiX52T+/PkyYMAA9UjeHRZz7miuZIgXexk0aJD07t1bateuLXXr1lWlxHfu3JE+ffqo11FmXKhQIZU7gj4olStXTrJ9eHi4+pt8OSHeJEzgwcR3AD1/KEwIIV4hTq5du6ay/lMDHg30QXEEzIgKb8ywYcNUEiw6zS5btiwxSfb06dP8sSUkFfbu3Svz5s2TKlWqSNu2bfldIYR4z8R/uCMrXLiwEg22aNmypUo0xR2c0WGfE+IpIMQJryaFCSHEKyf+QzhnxYoV0qpVK/UXSal4IAn2ySefVImxr7zyitOMJIRkTJigmzOFCSHEK8M6ECdIfv3ss8+UIEmed4KQDHJPCCGuZ/fu3aojcrVq1VTlGoUJIcQrwzrWTdNWrlwpp06dUs+LFSumurlmtOeIEWBYh5gZzF8FYYLcLFSqOTLNBCGEmLJ9vTV3795VJbqY5+btt9+WLl26OM0YQkjGQbn8H3/8ITVq1FAeEwoTQoinYLf/N2vWrKrBWbZs2VxjESEkXShMCCGejEPBabTBRh+TTESECCEOsmPHDiVMatasSWFCCPFIHEqIRSgHSbHoxNqvXz8pXry4ZMmSJcV6+PEkhDgPdGL+888/VSdlVMZRmBBCPBGHEmKtqwFs/Thil1jurgkAMwMTYolZwJQRixYtUt2TUcZPYUII0RvDJMSCSZMmOc0AQkjGhUmdOnXkiSeeoDAhhHg0DokTzH1DCHEPmEhz8eLFFCaEEK/BIXFiDZqxoTssQO5J3rx5nWEXIUREtmzZIkuWLFGTYGJaCAoTQog34HAryVWrVqnYd4ECBaRBgwbqgf9jGRqzEUIyx+bNm5UwqVevHoUJIcSrcMhzgjk8OnXqpGYKfuedd6Rs2bJq+aFDh2TKlCnK9Txr1ixp166ds+0lxGuEydKlS6V+/fry+OOPU5gQQrwKh6p1KlWqpObQWbt2rYSEhKTI3G3cuLGq1OGsxITYz6ZNm9SM3xQmhBCjY6hZiY8fPy59+vRJIUwAjOvbt6+cOHHCGfYR4lVs3LhRCROESSlMCCHeikNhnfLly6tE2NS4dOlSYqiHEJIxNmzYICtWrJCGDRuqCTQpTAgh3opDnpMvvvhCxo8fr1po28pH+fHHH+Wrr75yhn2EeJUwadSoEYUJIcTrcSjnBFOzHz58WI4cOSIFCxaU0qVLq+VHjx6V8+fPK69JmTJlkh7Ix8emmNEb5pwQvVm/fr389ddfKlerWbNmFCaEENPgqjHUIXGCfib2/oBifeSqGA2KE6In//33nyq9b9KkiZqrisKEEGImDNW+Xmu6RghxnHXr1ql+QRQmhBDi5A6xhBD7QRn+33//LU2bNpWHH36YwoQQQqygOCFEJ2Hy0EMPKWFCCCEkKRQnhLiRf//9V1avXk1hQgghaUBxQoib+Oeff2TNmjVKlECcEEIIsQ3FCSFuAKIE4gSJr8gzIYQQkjoUJ4S4EFTqQ5ggnIMeJqjMIYQQ4kJxcu7cOfWji1b2HTp0kMKFC6sJ/1DvjLpnPz+/zOyeEI8RJo8++qhqskYIIcRF7evxozto0CApUaKEdO/eXf0fHWPB7du3VZO2MWPGOLJrQjwCfEeQ+Aphgnb0FCaEEOJicfLll1/Kd999J2+99ZZqu23dZBYek/bt28vcuXMd2TUhpgffB5QKo2QYwgTz5RBCCHGxOJkwYYL06tVLPv30U6levXqK16tWrZroSSHE24QJur6i++tjjz1GYUIIIe7KOTlz5oya1j01smXLpvrtE+KNwgTz5Tz++OPSoEEDvU0ihBDvESd58+ZVAiU1tm3bJkWLFs2MXYSYTphgAj/MMNyiRQupX7++3iYRQoh3hXWQUzJ+/Pgkswxrc4OsWLFCJk+eLJ06dXKelYQYXJgg9wrCpGXLlhQmhBCSSXws1tmsGQSlwmgkdeLECdW3YdmyZSq+jkqdDRs2SI0aNVSVQtasWcVbp3sm3gG+PhDkGzduVMKkXr16eptECCGmH0Md8pzAEPwYv/POO6rXSXBwsOp+GRERIcOHD1dVCmYQJoRkVpgsX75cfReeeOIJChNCCNFTnIAsWbLI+++/Lzt37pQ7d+7IvXv3ZO/evTJs2DD1WmYYN26c6pUC0YMf/M2bN6e67rx586R27doSHh6uEnFRPTRlypRMHZ+QjAqTTZs2SatWraRu3bp6m0QIIR6Dw+LEVcycOVM1dYMHZvv27VKtWjWVYIgutLbImTOnDB06VIWTdu/eLX369FEPDByEuEqYIJQJYfLkk09KnTp19DaJEEI8CodyTp577rn0d+zjIxMnTrTbIHhK8GM/duxY9Tw+Pl6KFCkiAwcOlMGDB2doHzVr1lSDxkcffZTuusw5IfaAr8vSpUtly5Yt6hqD144QQryVSBeNoQ6VEqP7pVado4E5dS5cuKD+5smTR4VY7CU6OlqVIQ8ZMiRxma+vr+qyCc9IRjtzHjp0SD7//HOb60RFRamHBvuxkIyC62vJkiWydetWad26tdSqVUtvkwghxCNxSJycPHnS5vKYmBj58ccfZfTo0aq00l6uXr2qxE2+fPmSLMfzgwcPprodFFuhQoWU6MBkg99//72qHrLFqFGjZOTIkXbbRrwba2HSpk0b5Z0jhBBigpyTgIAAeeWVV1R3TPx1FyEhISoxF672Tz75ROWsYDZYW8ArAzGjPdJqJkeIJkwWL16shMlTTz1FYUIIIUb0nKQHklgdqZjJnTu38nxcunQpyXI8z58/f6rbIfRTunRp9X9U6xw4cEB5SB5++OEU6wYFBakHIRkVJosWLVLJ2RAm6OFDCCHEhNU6COk40uckMDBQxfExP4kGEmLx3J55SrCNdV4JIY4Kkz///FMJk7Zt21KYEEKIkT0nH374oc3laMKGzrD4Mc9oZU1yEJLp3bu3qoJA7wjkr6CPCsqDAWZDRn4JPCMAf7FuqVKllCBBXgC8Nj/88INDxyfEWpjs2LFDnn76aeUNJIQQYmBxMmLECJvLc+TIoUQC5t3p16+fQwZ17txZrly5opq5Xbx4UYVp0FNCS5I9ffq0CuNoQLgMGDBAzp49q5q/lS9fXqZOnar2Q4ijwmThwoWya9cuadeunVStWlVvkwghxKtwqM+JJ8E+JyR5SBAeEwgTeEwoTAghxARz66BNPUIv+AEnxNOECT0mhBCiP3aLE4RO0MskeUUNIWYXJn/88YeaAqF9+/ZSpUoVvU0ihBCvxaGcE1TUYJI/QjxFmCxYsEBd0xAmlStX1tskQgjxahwqJUYFzYwZM+Tnn3+W2NhY51tFiA7CpEOHDhQmhBBipoRYlAhXqFBBzZsDl/e1a9dUaAcNzVDai3BPkh37+KjYvdFhQqx3C5P58+fLvn37lDCpVKmS3iYRQoip0H3iv0ceeUSV6Hbt2lVy5cqlurmWK1fOaYYQ4m5hMm/ePNVNuGPHjlKxYkW9TSKEEGKvOIGDRXOypDZvDSFmFCbwCBJCCPHwuXUIMSqY9RrCBLNcU5gQQogHiBPkkRBiZmEyd+5cOXTokHTq1El1EyaEEGLihFi0jLdHnGBdM1TyMCHW+4TJM888w3wpQgjxhIRY0Lx5cylbtqzTDk6Iu4TJnDlz5PDhwxQmhBBiAuwSJ5gtuFu3bq6zhhAXCJPZs2fL0aNH1WSQFNeEEGJ8mBBLvEaYlClTRm+TCCGEZACKE+KRIN8JwuTYsWMUJoQQYjIoTohHCpNZs2bJ8ePHpUuXLlK6dGm9TSKEEOIKcYLGVYSYRZicOHFCdTMuVaqU3iYRQgixE3pOiEcJk5kzZ8rJkyeVx4TChBBCzAnFCfEYYYKZsk+dOqU8JiVLltTbJEIIIQ5CcUJMT0xMjPKYUJgQQohnQHFCTC9M4DE5ffq06sFTokQJvU0ihBCSSShOiOmFyZkzZ6R79+5SvHhxvU0ihBDiBChOiGmFye+//y5nz55VHhMKE0II8RwoTojpiI6OVsLk3LlzymNSrFgxvU0ihBDiRHyduTNCXA2FCSGEeD70nBBTCZPp06fLhQsXpEePHlK0aFG9TSKEEOIC6DkhphMm8JhQmBBCiOdCzwkxPFFRUUqYXLx4UXlMihQpordJhBBCXAjFCTG8MJk2bZpcvnxZevbsKYULF9bbJEIIIS6GYR1iCmECjwmFCSGEeAf0nBDDCpOpU6fKlStXlMekUKFCeptECCHETVCcEMNx//595TGhMCGEEO+E4oQYTpjAY3Lt2jXp1auXFCxYUG+TCCGEuBnmnBBDChN4TChMCCHEO6HnhBhGmEyZMkWuX7+uPCYFChTQ2yRCCCE6QXFCdOfevXvKY3Ljxg0KE0IIIcYM64wbN07NMhscHCz16tWTzZs3p7ruhAkTpEmTJpIjRw71aN68eZrrE+MJE3hMKEwIIYQYVpzMnDlTBg0aJMOHD5ft27dLtWrVpEWLFqrXhS3WrFkjXbt2ldWrV8uGDRtU99DHH39cTQxHjC9MfvvtN4mIiFDCJH/+/HqbRAghxAD4WCwWixgIeErq1KkjY8eOVc/j4+OV4Bg4cKAMHjw43e3j4uKUBwXbY8BLj8jISAkLC5ObN29KaGioU94DSZ+7d+8qjwnOPz6nfPny6W0SIYQQO3HVGOprtMndtm3bpkIzGr6+vuo5vCIZHfRiYmIkZ86cqTb3wsm0fhD3gs8IHhMKE0IIIYYXJ1evXlWej+SDFZ5j0reM8O6776oSVGuBY82oUaOUytMenEROH2Fy69Yt6d27N4UJIYQQY4uTzPLZZ5/JjBkzZP78+SqZ1hZDhgxR7iftcebMGbfb6a3cuXNHfv31V7l9+7YSJnnz5tXbJEIIIQbEUKXEuXPnFj8/P7l06VKS5XieXrLkV199pcTJypUrpWrVqqmuFxQUpB7E/cIEHhP8hTDJkyeP3iYRQggxKIbynAQGBkqtWrVk1apVicuQEIvnDRo0SHW7L774Qj766CNZtmyZ1K5d203WEns9JhQmhBBCTOc5ASgjxgAGkVG3bl0ZPXq0GtT69OmjXkcCJSaCQ+4I+Pzzz2XYsGEyffp01RtFy03Jnj27ehB9QQgHHhOUDT/77LPKO0YIIYSYSpx07txZzUYLwQGhUb16deUR0RInT58+rSp4NH744QdV5dOxY8ck+0GflBEjRrjdfpJUmMBjgtb0EJwUJoQQQkzZ58TdsM+Ja4UJSrchTHLlyqW3SYQQQkwyhhrOc0LMD8qEEcqhMCGEEOIIFCfE6cIEHhOE2ihMCCGEOALFCXG6MEGHXiS/ptallxBCCEkLihPitLgjhElsbKzymFCYEEIIcRSKE+I0YYKpB+AxwcSLhBBCiKNQnJBMgQxtCBM0y4PHhMKEEEJIZqE4IU4RJvCYhIeH620SIYQQD4DihDhERESEEiaAwoQQQojHzq1DzCdMEMqhMCGEEOJM6DkhDgkTHx8fJUzQGZAQQghxJvSckAxz48YNmTx5MoUJIYQQl0LPCcmwMIHHBJMuIseE8xARQghxFfSckAx7TPz8/ChMCCGEuBx6TkiaXL9+XXlM/P39VSiHwoQQQoiroTgh6QqTgIAAJUxCQkL0NokQQogXwLAOscm1a9dUKIfChBBCiLuh54TYFCbwmAQFBUmvXr0oTAghhLgVihOShKtXryphEhwcrDwm2bNn19skQgghXgbFCUkhTLJkyaI8JhQmhBBC9IA5J0RBYUIIIcQo0HNC5MqVK0qYZMuWTQkT/CWEEEL0gp4TL4fChBBCiNGg58SLuXz5shImqMbp2bMnhQkhhBBDQHHipVgLE3hMsmbNqrdJhBBCiILixAu5dOmS/Pbbb6oVPTwmFCaEEEKMBHNOvFCYwGNCYUIIIcSo0HPiRVy8eFF5TMLDw5UwQdkwIYQQYjToOfESKEwIIYSYBXpOvIALFy4oYZIzZ07p0aMHhQkhhBBDQ3HiRcIEHhPMmUMIIYQYGYoTD+b8+fMyZcoUyZUrl/KYUJgQQggxA8w58VAoTAghhJgVek48kHPnzilhkidPHiVMgoKC9DaJEJIKcXFxEhMTo7cZhKRKYGCg+Pq615dBceJhnD17VqZOnUphQojBsVgsqoouIiJCb1MISRMIkxIlSiiR4i4oTjxQmOTNm1e6d+9OYUKIgdGECb6vaIbo4+Ojt0mEpCA+Pl6lCaC4omjRom67Tg0nTsaNGydffvml+uJWq1ZNxowZI3Xr1rW57r59+2TYsGGybds2OXXqlHz77bfy+uuvizdy5swZJUzy588v3bp1ozAhxOChHE2YIC+MECOTJ08eJVBiY2MlICDA+xJiZ86cKYMGDZLhw4fL9u3blThp0aKFmqTOFnfv3pWSJUvKZ599pgZlb4XChBBzoeWYcPoIYgYCH4RzIKrdhaHEyTfffCP9+vWTPn36SMWKFWX8+PHqy/vLL7/YXL9OnTrKy9KlSxevHZBPnz6thEmBAgUYyiHEZDCUQ8yAjw7XqWHESXR0tArPNG/ePEkSDp5v2LBBV9uMLEymTZumhAk8Ju5MViKEEEI8XpxcvXpVuYzy5cuXZDmeI//EWURFRUlkZGSShxlBjg08JgULFqQwIcSbiY8TubRG5OTvCX/x3IQUL15cRo8erbcZxCAYLiHW1YwaNUpGjhwpZgbCBB6TwoULS9euXd2WoEQIMRhn5olse03k7tn/X5a1sEit70SKtNfTMkI8w3OSO3du8fPzk0uXLiVZjufOTHYdMmSI3Lx5M/GBZFIzcfLkSQoTQkiCMFnbMakwAXfPJSzH6wYJ2RNiWnGCsEStWrVk1apVSeqr8bxBgwZOOw4SRkNDQ5M8zMKJEydk+vTpUqRIEQoTQrwZhG7gMRGLjRcfLNv2uktCPA8//LC88sor6hEWFqZuLD/44APVVE4Lz3z00UfSq1cv9fvav39/tXzu3LlSqVIl9RuMdb7++usU+75165b6bcuWLZsUKlRItZZIXjRRpUoV9Tp+BwcMGCC3b99O4lVu06aN5MiRQ62D4y1ZssTp54B4WVgHZcS9e/eW2rVrq94miD/euXNHVe8AXOy4YBGa0RT5/v37E/+Ptu07d+6U7NmzS+nSpcWT0IQJmuCgOonChBAPJPauSOTB9Ne7tjWlxyQJFpG7Z0SOTRTJVTv9/YWWF/HPeFnzr7/+Kn379pXNmzfL1q1blQDBbxOqLcFXX32lelChLQRAscMzzzwjI0aMkM6dO8v69euVsECPl2effTZxv6i+fO+991Toffny5fLaa69J2bJl5bHHHksskvjf//6nupUeP35c7eOdd96R77//Xr3+8ssvq7Hg33//VeIE4wPGA2I+fCya3DUIY8eOTWzCVr16dXUh1qtXL1GxQ3FPnjw5McSBizQ5Dz30kKxZsyZDx0NCLNQ/QjxG9aLgS/j7779LsWLF1BebwoQQc3P//n11w4HfrySTcl7fLrKslvsNarlNJGfNDK2K32H0nkITTK3EdPDgwbJw4UIlBvAbXaNGDZk/f37iNmhzcOXKFVmxYkXiMoiKxYsXq/0AbFehQgVZunRp4jq4EcNvdGrejzlz5siLL76oCipA1apVpUOHDomiiLj4enXhGGoozwnQ3IW2SC44cDEbTFu5TJjgvUKY+Psb7iMjhDgLeDAgFDLiOdnyQvrr1fkx454TO6hfv36S3hcIvSNMozXpgvfbmgMHDkjbtm2TLGvUqJHyjmMb5Btq+7EGz60reFauXKk85wcPHlSDIjqWYuBEQ070xHr11VflpZdeUiIIbSggVCBYiPngSGdgjh07JjNmzKAwIcRbQGglIx6M8Goi+z5KSH61mXfik1C1U6qviG/CwO9OEFJxNvCUt27dWomPTz75RHLmzCnr1q1T4SWEciBOnn/+edVVHB4ZCBQIGYimgQMHOt0e4iUJsSQpR48eVR4TuNEoTAghSYDgQLmwInn3zgfPa412mTDZtGlTkucbN26UMmXKJHpAkoNwzX///ZdkGZ4jn8R6G+wn+X6xrZa3giIJiA14brAt5ntJDhJlEeqZN2+evPnmmzJhwoRMvVeiDxQnBhUm8Jhg3iAkkVGYEEJSgD4mTeaIZC2UdDk8Jljuwj4n6E6NAoZDhw6pmyhM0Irk1dSASEDlJap4Dh8+rBJqkV/41ltvpRAsX3zxhVoHlTqzZ89O3C+KHDAnEY6FcPeUKVPUFCfWYOJXJNIiPwLzs61evTpR3BBzwVHPYBw5ckRNgFiqVCnp1KkThQkhJHUgQAq1FbmyVuTeBZEsBUTyNHF5KAeVk/fu3VNVlfB8QEBoJcO2qFmzpsyaNUtV8ECgYMqNDz/8MEmljiZiUP2Dah0kV6J0GGEagIlg8fzzzz9X/aqaNm2qwjawRQP5K6jYOXv2rNq+ZcuWarZ6Yj4MV63jboxUraMJE9whQJik5iIlhHhu9YPRQbUOKinZat57uM9qHe8FbkzcWVCYEEII8XYoTgwA4rYQJkjw6tixI4UJIYQQr4bixCDCpFy5cqomn8KEEGJkMtrgkpDMQHGiI2gkhGx0ChNCCCHk/6E40Ql0TETr5fLly0v79u0pTAghhJAHUJzoKExQfw9hgsmsCCGEEJIAxYmbwcRYECaYyrtdu3YUJoQQQkgyKE7cCGbfnDt3LoUJIYQQkgYUJ25CEyaVK1eWp59+msKEEEIISQWOkG5g7969SphUqVKFwoQQ4lTi4+Ll5JqTsuf3PeovnhuZESNGqA6zGmhhj99F6w60mCPH6CR/H+5k8uTJEh4eLp4MPScuZs+ePTJ//nwlTNq2bUthQghxGgfmHZBlry2TyLORictCC4dKy+9aSoX25pjw7rvvvhN3zaICQbFgwQLZuXOnW45HHIcjpRuESdWqVSlMCCFOFyazOs5KIkxA5LlItRyvmwHMy5JZL0B0dLR4Api4MD7e2J4vd8HR0kXs3r1bCRPMpPnUU09RmBBCnAZCN/CYiC2Hw4Nly15f5pIQDwbPL774Qs0DFhQUJEWLFpVPPvkk8fV3331XTcWRNWtWKVmypHzwwQcSExOT6v6Sh3VAbGysvPLKK0q45M6dW+3D2rtSvHhxNbsxZiTGZHPajMhpHRuhEMx2vGvXLvHx8VEPLAMRERHy/PPPS548edT+mjVrptaz5rPPPpN8+fJJSEiI9O3bV02Gl14nXRxj8eLF6gYVE+bVr19fhfmTh2cWLlwoFStWVOfz9OnTcuPGDfXecuTIod7LE088oSaGTQ68QGXKlFH7xuzNZ86cSXzt2LFj6qYYNmfPnl3q1KkjK1euTLL9999/n7g91sP0KUaBYR0XgIsaFw3ikRAmuEAJISQ9Yu7GyNWDV9Nd7/zW8yk8JkmwiESeiZQdE3dIwdoF091f7vK5JSBrQIZsHDJkiEyYMEG+/fZbady4sVy4cEF1u9bA4I1Bt2DBgsp73K9fP7XsnXfekYzy66+/KgGwefNm2bp1qxIfEEHYl8ZXX30lw4YNk+HDh2fo2J07d1bCYNmyZYmDNMQPwGSrWbJkkaVLl6plP/74ozz66KNqQtacOXOqKUYQEho3bpx6z1OmTJH//e9/SgClx9tvv61CV/nz55f33ntP2rRpo/YbEJBwvu/evSuff/65/Pzzz5IrVy7JmzevdO3aVYkRiBaIJYiuVq1aqVYU1ttBFP72228SGBgoAwYMkC5dush///2nXr99+7baButA9GA9HBtTpuBc4ry++uqr6r00bNhQrl+/LmvXrhWj4GNxV7DPoDh7umfEMv/44w+pUaOGuhAoTAghGZ2C/sL2C/JTrZ/cbk//bf2lQM0C6a5369Yt5V0YO3as8jRkBIiIGTNmqMHQVt4HPCfwXGCZlhB7+fJlVeGo/X4OHjxYDdQYnDXPCX5j4Z3OzLHBunXr5Mknn1THxCCuAc8QRA2EEQZvHA/iRANeEHyOqeWvwHPyyCOPqONDGAEIgMKFCysB9cwzz6i/ffr0UfuAlx1AlJQtW1aJDBwXXLt2TYoUKaJEG4SUtt3GjRulXr16ah0IRDT23LRpk9StW9emTagWffHFF5VXat68eWofZ8+eVQLOkevVFWOoBj0nToTChBCSGeDBgFDIiOdk0QuL0l2v9Y+tM+w5yWh366ioKOVVSI2ZM2cqrwLCCrh7R4jG3kELA7/172eDBg3k66+/VjkZ2lQftWvXdsqx4enGuvBaWHPv3j21H+19Y1C3BjatXr063feC9TTghcFcatifBrweCPto4DV/f/9E0QFgW/LtsA5CNRqYCgUhIqwDcYL3BDGGsBK8WzgXeE8IG4HHHntMihUrprw/LVu2VA/030IYyQhQnDiJHTt2KGVfs2ZNad26NYUJIcRuEFrJiAcjX7V88u9H/6rkV5t5Jz4JVTs1+tYQXz/n5bsh9JEWGzZskO7du6vcDuRA4I4angMIC2eTLVs2pxwbg3iBAgVszrbsjnJdnFNXjBdvvfWW/PXXX8p7BC8QjoOcEi15GN6S7du3q/e9YsUKFSKDmNmyZYshypSZpekE8AFDmNSqVYvChBDiciA4UC6sSP5z8+B5y9EtnSpMAJInMcitWrXK5uvr169Xd+NDhw5Vng2sf+rUKbuPg9CENQhfYF9pTZCakWPDSwHvizW4obx48aLyRGAQt34gGRdo4ZLkNmUE6/WQ6Ip8E+wvNfBabGxskuMhrINcESTNamAdLVwF8DrCY9q+ERZCyAzeELSyQM7LyZMnkxwL77l58+YqwRlFHHj977//FiNAcZJJtm3bJn/++af6MiBuSWFCCHEH6GPyzJxnJLRQ0rAFPCZY7oo+J8g3QHImcjGQYImwBwbfiRMnqtchCBA2gMcCryHEkl5eiC2wj0GDBqkB9/fff5cxY8bIa6+9luY2GTk2clWQO4EQ/NWrV1WICoMzQi+oGIIHAQM0hA5Ejjb449i//PKLTJo0SYkLJOEiJyYjfPjhh0rMIRkXYgGCJ3l1UvL30bZtW5XMi3wYhJ169OghhQoVUss1kBg7cOBAJWIwDmHfCIdp+SbYD/JK8F6xj27duiUpU160aJE6R3gdIg6fJ15H+MgQWLycmzdvwimq/trL1q1bLSNGjLAsXrzYEh8f7xL7CCGex7179yz79+9XfzNLXGyc5cTqE5bd03erv3juSuLi4iwff/yxpVixYpaAgABL0aJFLZ9++mni62+//bYlV65cluzZs1s6d+5s+fbbby1hYWGJrw8fPtxSrVq1xOe9e/e2tG3bNvH5Qw89ZBkwYIDlxRdftISGhlpy5Mhhee+995L8xuLY2G9y0jv2/fv3LR06dLCEh4er3/1Jkyap5ZGRkZaBAwdaChYsqN5TkSJFLN27d7ecPn06cdtPPvnEkjt3brVv2PzOO+8keR/JWb16tTrGn3/+aalUqZIlMDDQUrduXcuuXbsS18Hxre3TuH79uqVnz57qtSxZslhatGhhOXz4cIrt5s6daylZsqQlKCjI0rx5c8upU6cS1zlx4oTlkUceUdvj/YwdO1ad29dee029vnbtWvUc5xfrVK1a1TJz5ky7r9fMjKFpwWodBzONoaiRaISEJNSg02NCCMkoaVU/EM9Aq9ZBKMcIORyZgdU6JgEJQ0uWLFHuM2Q4U5gQQgghzoPixE7QFAiNelDmhYxwChNCCCHEuVCcOCBMkHT0+OOPU5gQQgixCZrJeXnWRKagOMkgyIhG22MKE0IIIcS1UJxkAJTKLV++XJWboasehQkhxBnwzpqYAYsO1yn7nKQDug5CmGCOAwoTQogzsJ68jRCjE/2gq2xaTfCcDT0n6QgTNOVp1KiRmkuCwoQQ4gzwI4/yUkw2BzCfCX9fiBGJj4+XK1euqGsUHWXdBcVJKqBDIOYlwPTYzZo14w8HIcSpoJ040AQKIUbF19dXihYt6tZxkOLEBpiTYOXKldKkSRPVRIfChBDibPC7ggnn8ubNKzExMXqbQ0iqYE4iCBR3QnGSDMxlgHkQKEwIIe4K8bgzlk+IGTBkQuy4cePUBE1ok4tmZ+gvkhazZ8+W8uXLq/Ux+yK6t2ZGmDRt2pTChBBCCNEJw4mTmTNnqtkoMevj9u3bpVq1aqoTa2pxWeSGdO3aVfr27Ss7duxQsz3igRkg7QH7gTB56KGHKEwIIYQQHTHcxH/wlGAyvbFjxyZmChcpUkRNDT148OAU63fu3Fnu3Lmjpn/WQKO06tWry/jx49M9njZpEfYNEYSufoQQQghJH6+Y+A+11Nu2bZMhQ4YkLkMSTvPmzVVZry2wHJ4WayAyFixYYHP9qKgo9dDACQW1atWSmjVrqhNNCCGEkPTRxkxn+zkMJU6uXr0qcXFxki9fviTL8fzgwYM2t7l48aLN9bHcFqNGjZKRI0emWN6pU6dM2U4IIYR4K9euXVMeFI8UJ+4AXhlrT0tERIQUK1ZMTp8+7dQT6y7FipDXmTNnnOpOczVmtdvMtpvVbjPbbla7zWy7We02s+03b95UPVBy5szp1P0aSpzkzp1bldRdunQpyXI81xoWJQfL7Vk/KChIPZIDYWKmC8Ia2G1G281qt5ltN6vdZrbdrHab2Xaz2m1m232d3AfF12iNXpD7gaoZDSTE4jkm3bMFlluvD9DZNbX1CSGEEGJsDOU5AQi59O7dW2rXri1169aV0aNHq2qcPn36qNd79eolhQoVUrkj4LXXXlPlv19//bU8+eSTMmPGDNm6dav89NNPOr8TQgghhHiEOEFpMCYZGjZsmEpqRUnwsmXLEpNekRti7T7CbMHTp0+X999/X9577z0pU6aMqtSpXLlyho6HEA96qtgK9Rgds9puVrvNbLtZ7Taz7Wa128y2m9VuM9se5CK7DdfnhBBCCCHejaFyTgghhBBCKE4IIYQQYigoTgghhBBiKChOCCGEEGIovEKcjBs3TooXLy7BwcFqYsHNmzenuf7s2bOlfPnyav0qVarIkiVLxAy279u3Tzp06KDWx6zKKMM2g90TJkyQJk2aSI4cOdQDcyml9xkZxfZ58+apsvfw8HDJli2bqi6bMmWKmOE610D5Pa4XzOatF/bYPnnyZGWv9QPbmeGcoyP1yy+/LAUKFFDVDWXLltXt98Ue2zEhavJzjgfaNxj9nON3sFy5cpIlSxbVgfWNN96Q+/fvix7YY3tMTIx8+OGHUqpUKbV+tWrVVOWqu/n333+lTZs2UrBgQfWZpzZvnTVr1qxRc9XhGi9durT6ztqNxcOZMWOGJTAw0PLLL79Y9u3bZ+nXr58lPDzccunSJZvr//fffxY/Pz/LF198Ydm/f7/l/ffftwQEBFj27NljeNs3b95seeuttyy///67JX/+/JZvv/3Wogf22t2tWzfLuHHjLDt27LAcOHDA8uyzz1rCwsIsZ8+eNbztq1evtsybN09dK0ePHrWMHj1aXT/Lli0ztN0aJ06csBQqVMjSpEkTS9u2bS16YK/tkyZNsoSGhlouXLiQ+Lh48aLh7Y6KirLUrl3b0qpVK8u6devUuV+zZo1l586dhrf92rVrSc733r171XWOz8LIdk+bNs0SFBSk/uJ8L1++3FKgQAHLG2+84Va7HbH9nXfesRQsWNCyePFiy7Fjxyzff/+9JTg42LJ9+3a32r1kyRLL0KFD1e8cJMP8+fPTXP/48eOWrFmzWgYNGqR+F8eMGePQb6LHi5O6detaXn755cTncXFx6gMfNWqUzfWfeeYZy5NPPplkWb169SwvvPCCxei2W1OsWDHdxElm7AaxsbGWkJAQy6+//moxm+2gRo0aStQa3W6c54YNG1p+/vlnS+/evXUTJ/bajgER4lVv7LX7hx9+sJQsWdISHR1t0ZvMXuf4bcF39Pbt2xYj2411mzVrlmQZBs1GjRpZ3I29thcoUMAyduzYJMvat29v6d69u0UvMiJOIKoqVaqUZFnnzp0tLVq0sOtYHh3WiY6Olm3btqkwgQYauOH5hg0bbG6D5dbrgxYtWqS6vpFsNwLOsPvu3bvKpensiaRcbTu+u5hK4dChQ9K0aVMxut1wGefNm1f69u0reuGo7bdv31YTdsJN37ZtWxXSNLrdCxcuVNNqIKyDppJoFPnpp5+qmdjN9h2dOHGidOnSRYUyjWw3mnRiGy18cvz4cRVGa9WqlbgTR2yPiopKEa5EaGrdunViZJw1hnq0OLl69ar64mvdZTXwHN1nbYHl9qxvJNuNgDPsfvfdd1V8M/kFblTbMStn9uzZ1dxQiMGPGTNGHnvsMTGy3fiBwwCDfB89ccR25A/88ssv8scff8jUqVPV/FsYhM6ePWtouzEwzpkzR22HAfKDDz5Q0258/PHHYqbvKAb6vXv3yvPPPy9Gt7tbt25KhDdu3FgCAgJU/gbyZ9BN3Oi2t2jRQr755hs5cuSIusYxZxxy3C5cuCBGJrUxFLMu37t3L8P78WhxQszHZ599phI058+fr1uSo72EhITIzp07ZcuWLfLJJ5+o+aGQEGZUbt26JT179lTCBDOBmw14HzDHFpKPMa8WfrDz5MkjP/74oxgZDDDwVGHeL0xwiqk6hg4dKuPHjxczAVGLQgHMfWZ08D2Ed+r777+X7du3q2tl8eLF8tFHH4nR+e6779R0LCjOwI3PK6+8ouaYc/bsv0bFcHPrOBP88Pr5+cmlS5eSLMfz/Pnz29wGy+1Z30i2G4HM2P3VV18pcbJy5UqpWrWqmMV2/FggIx1gwDxw4ICamBJ3aEa0+9ixY3Ly5EmVgW89cAJ/f38VlsIdplmuc9wR16hRQ44ePSruwhG7UaEDW7GdRoUKFdSdJtz+GICMfs4xCStuHuCNcDeO2A3vFIS45uWBqMJ76N+/vxKG7hroHbE9T548qjIGlUXXrl1T3uTBgwdLyZIlxcikNoaGhoaqsFRG8WgJhi877lCQB2D9I4znuPuyBZZbrw/gTkttfSPZbgQctfuLL75QdzMolUNprh4465xjG8SLjWo37sT27NmjvD3a46mnnpJHHnlE/R95HEa13RZwl+P9YPA3st2NGjVSAkoTguDw4cPKbncJk8yec7RZwLXdo0cPcTeO2I38teQCRBOH7pxWLjPnPDg4WAoVKiSxsbEyd+5clWNlZJw2hlo8HJRvoZRs8uTJqqypf//+qnxLKz3s2bOnZfDgwUlKif39/S1fffWVKmsdPny4rqXE9tiOUkWU4+KBTG+UFeP/R44cMbTdn332mSqxmzNnTpJyxVu3brnVbkds//TTTy0rVqxQpX5YH9cNrp8JEyYY2u7k6FmtY6/tI0eOVCWhOOfbtm2zdOnSRZVYojzTyHafPn1aVbi88sorlkOHDlkWLVpkyZs3r+Xjjz92q92O2K7RuHFjVXmhF/bajd9vnHO0V0CJK76rpUqVUlWZRrd948aNlrlz56rr/N9//1VVRyVKlLDcuHHDrXbjd1gbVyAZvvnmG/X/U6dOqddhM2xPXkr89ttvqzEUbSJYSpwKqLMuWrSoGgBRzoUPXeOhhx5SP8zWzJo1y1K2bFm1PkqiUGduBttRx4+LJ/kD6xnZbpQ927IbPyx6YI/tqP8vXbq0Ghxz5MhhadCggfoRMrrdRhIn9tr++uuvJ66bL18+1TfE3b0fHLEbrF+/XrUmwCCFsuJPPvlElXSbwfaDBw+q7yUGeD2xx+6YmBjLiBEjlCDBd7RIkSKWAQMGuH2Ad8T2NWvWWCpUqKCulVy5cikBcO7cObfbjF5Otn6fNVvxN/kYg22qV6+u3ieuc0f64fjgH+c6dQghhBBCHMejc04IIYQQYj4oTgghhBBiKChOCCGEEGIoKE4IIYQQYigoTgghhBBiKChOCCGEEGIoKE4IIYQQYigoTgjxQIoXLy7PPvtskgnQfHx8DDUhYXIbjQDOESZYcxaTJ09W+9y6dWu662IuJuv5mDD/EbbFPjRGjBihlhHi6VCcEOJktAFJe2BujLJly6pBL/mEWEZnyZIlakAkxgWz7mKCOEI8CYoTQlwEZm6dMmWKjB07Vho2bCg//PCDmvwKk5G5m6ZNm8q9e/fUX3vFyciRI11mF/l/VqxYoR5p8f7776vP0RqKE+KJ+OttACGeyhNPPJE4wzKmbM+VK5d888038scff0jXrl1tboPp3LNly+Z0WzAzKzw43oarzqcryMjMxP7+/upBiKdDzwkhbqJZs2bq74kTJ9Rf5Ftkz55djh07Jq1atZKQkBDp3r174nTqo0ePlkqVKilRkS9fPnnhhRfkxo0bSfaJqbE+/vhjKVy4sGTNmlUeeeQR2bdvX4pjp5ZzsmnTJnXsHDlyqEG8atWq8t133yXaN27cOPV/6zCVhrNttIWWd/HVV1/Jt99+K8WKFZMsWbLIQw89JHv37k2yblrnEyLlzTfflCJFikhQUJCUK1dO7TO1qcWmTZum1sH7wlT3//77b5LXT506JQMGDFDrwB4Iz06dOil7bQFvGc4N1gsNDZVevXqlOE/Jc05skTznBP/He/v1118TPx+ch9WrV6v/z58/P8U+pk+frl7bsGFDmsciRE8owQlxExg0AQYojdjYWGnRooU0btxYDZYYvAEGMuSu9OnTR1599VUlaBAe2rFjh/z3338SEBCg1hs2bJga+DEY47F9+3Z5/PHHJTo6Ol17/vrrL2ndurUUKFBAXnvtNcmfP78cOHBAFi1apJ7DhvPnz6v1EJ5Kjjts1Pjtt9/k1q1b8vLLL8v9+/eVgILY27NnjxJFaZ1PCJCnnnpKDdh9+/aV6tWry/Lly+Xtt9+Wc+fOKdFjzT///CMzZ85U7wlC5vvvv5eWLVvK5s2bpXLlymqdLVu2yPr166VLly5KdEGUIGwHcbF///7Ez1ED+Ubh4eFKXBw6dEitC4GjiUZHwecCr1zdunWlf//+almpUqWkfv36SohBZLVr1y7JNliGdRBiJMSwOGtaZUJIApgeHF+tlStXWq5cuWI5c+aMZcaMGWra8yxZsljOnj2bONU41hs8eHCS7deuXauWT5s2LcnyZcuWJVl++fJlNSX5k08+aYmPj09c77333ksypbn1tOf4C2JjYy0lSpSwFCtWLMX08db7evnll9V2yXGFjbY4ceKEWs/6vIFNmzap5W+88UbistTO54IFC9Tyjz/+OMnyjh07Wnx8fCxHjx5NXKZNB79169bEZadOnbIEBwdb2rVrl7js7t27KWzdsGGD2va3335LcS3UqlXLEh0dnbj8iy++UMv/+OOPxGWYdt566nntvVtPNz98+PAUn0e2bNlsnschQ4ZYgoKCLBEREYnL8Hn4+/ur/RBiZBjWIcRFNG/eXPLkyaPuYHGHjZAD3OyFChVKst5LL72U5Pns2bMlLCxMHnvsMbl69WriA+EF7AMeALBy5UrlfRg4cGCSu+/XX389Xdvg3YCnA+vijt6ajNzJu8NGa55++ukk5w2egnr16qmE3eQkP59Yx8/PT3lCrEGYB3pk6dKlSZbDo4D3oVG0aFFp27at8rbExcWpZQjlaMTExMi1a9ekdOnS6lzCM5QceDU0T5JmI3JHbNnvLBA6ioqKkjlz5iQug0cI3qUePXq47LiEOAOGdQhxEcjXQAkxBiGEHpCfgMRUa/AawgLWHDlyRG7evCl58+a1ud/Lly+rvwgLgDJlyiR5HYIIOSQZCTFpYQp7cYeN1iTfHuDczpo1K93zCRsKFiyoclCsqVChQhIb0zsW8kauXLmiwl+omBk1apRMmjRJhYasc1dwXtKzHwIO4bTUclScQfny5aVOnToqjINwFsD/EfKBkCLEyFCcEOIicHevVeukBnIakgsWJJpi0MdAYgsM7HpjVBttnU9XAE8QhAk8QPC0wIsEzxA8ZDg3RgHeE+QPnT17VnlRNm7cqPKCCDE6FCeEGAwkKyIc0qhRoyThg+SgckXzYpQsWTJxOe7uk1eC2DoGQMULwk+pkVqIxx02WoPtk3P48GHVZTY9YANsRUKttffk4MGDSWxM71hIctVEF0IlvXv3lq+//jpxHSTqRkREpGo/qpQ0bt++LRcuXFAJwpklrTAcxNKgQYPk999/V94ehJY6d+6c6WMS4mqYc0KIwXjmmWdUbsNHH32U4jXkC2gDIEQFBpsxY8YkCSugvDc9atasKSVKlFDrJh9Qrfel9QhJvo47bLQGTcYQPtFA5QzKoNFLJj0gAGBrco8BqnQwsCffB0psrfNGzpw5o3rToMIIuSsAf5OXIeM9ajkpyfnpp59UbooGqnVwnjJif3rgM0pNFOXOnVsdY+rUqcrLhaojLCPE6NBzQojBQA8PlOkip2Hnzp1qUMQAj7tvJKKijLZjx47qLv6tt95S66EkGIMwEl2R4JneAITQBwbINm3aqNJalAMjBwLeBPQgQfIn0BJDkUyKEl0Myrgbd4eN1iBHAuXBSCRFeALiBiXZ77zzTrrb4j3CazF06FCV41GtWjXViRWCA2EZzYukgTwcvFfrUmJg3SkX7wVlvAjnVKxYUQkaeGesy8StQVLwo48+qkQdSomxT7wflDhnFnxGODYa/CG3BqITycLWoR18FsCWmCTEkOhdLkSIp6GVj27ZsiXN9VD+iTLQ1Pjpp59UCSrKaENCQixVqlSxvPPOO5bz588nrhMXF2cZOXKkpUCBAmq9hx9+2LJ3715VIpxWKbHGunXrLI899pjaP2ypWrWqZcyYMYmvo+R44MCBljx58qiy2+Q/Gc600RZaOe2XX35p+frrry1FihRR5bFNmjSx7Nq1K8Pn89atW6rsuGDBgpaAgABLmTJl1D6ty5sBjoXy6alTp6p1cKwaNWqkOG8ov+7Tp48ld+7cluzZs1tatGhhOXjwYIr3pF0L//zzj6V///6WHDlyqPW7d+9uuXbtWpJ9OlpKjOM2bdpUnVtb5dlRUVHquGFhYZZ79+6leb4JMQo++EdvgUQIIbaApwOegC+//FJ5YIj9IHwEjwo8SBMnTtTbHEIyBHNOCCHEg0G+DhKQEd4hxCww54QQQjwQJAzv3r1b5ZnUqFFD5QkRYhboOSGEEA8ECc9IIEY/GsxNRIiZYM4JIYQQQgwFPSeEEEIIMRQUJ4QQQggxFBQnhBBCCDEUFCeEEEIIMRQUJ4QQQggxFBQnhBBCCDEUFCeEEEIIMRQUJ4QQQggxFBQnhBBCCBEj8X9kSLgiZOmN6wAAAABJRU5ErkJggg==",
"text/plain": [
- ""
+ ""
]
},
- "metadata": {
- "needs_background": "light"
- },
+ "metadata": {},
"output_type": "display_data"
}
],
@@ -311,9 +344,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "/Users/nacho/notebooks/tsai/nbs/052c_calibration.ipynb saved at 2022-11-09 12:54:31\n",
+ "/Users/nacho/notebooks/tsai/nbs/021_calibration.ipynb saved at 2025-01-18 17:43:29\n",
"Correct notebook to script conversion! 😃\n",
- "Wednesday 09/11/22 12:54:34 CET\n"
+ "Saturday 18/01/25 17:43:32 CET\n"
]
},
{
diff --git a/nbs/029_models.layers.ipynb b/nbs/029_models.layers.ipynb
index 2870f7cf5..fbc0cae54 100644
--- a/nbs/029_models.layers.ipynb
+++ b/nbs/029_models.layers.ipynb
@@ -62,7 +62,7 @@
" verbose:bool=True, # If `True`, prints detailed information about the tracing and scripting process. Defaults to `True`.\n",
"):\n",
" \"Tests if a PyTorch module can be correctly traced or scripted and serialized\"\n",
- " \n",
+ "\n",
" m = m.eval()\n",
" m_name = m.__class__.__name__\n",
"\n",
@@ -131,13 +131,7 @@
"output_type": "stream",
"text": [
"output.shape: torch.Size([3, 2])\n",
- "Tracing...\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
+ "Tracing...\n",
"...Linear has been successfully traced 😃\n",
"\n"
]
@@ -167,11 +161,11 @@
"source": [
"#|export\n",
"def init_lin_zero(m):\n",
- " if isinstance(m, (nn.Linear)): \n",
+ " if isinstance(m, (nn.Linear)):\n",
" if getattr(m, 'bias', None) is not None: nn.init.constant_(m.bias, 0)\n",
" nn.init.constant_(m.weight, 0)\n",
" for l in m.children(): init_lin_zero(l)\n",
- " \n",
+ "\n",
"lin_zero_init = init_lin_zero"
]
},
@@ -181,9 +175,9 @@
"metadata": {},
"outputs": [],
"source": [
- "#|export \n",
+ "#|export\n",
"class SwishBeta(Module):\n",
- " def __multiinit__(self, beta=1.): \n",
+ " def __multiinit__(self, beta=1.):\n",
" self.sigmoid = torch.sigmoid\n",
" self.beta = nn.Parameter(torch.Tensor(1).fill_(beta))\n",
" def forward(self, x): return x.mul(self.sigmoid(x*self.beta))"
@@ -199,7 +193,7 @@
"class SmeLU(nn.Module):\n",
" \"Smooth ReLU activation function based on https://arxiv.org/pdf/2202.06499.pdf\"\n",
"\n",
- " def __init__(self, \n",
+ " def __init__(self,\n",
" beta: float = 2. # Beta value\n",
" ) -> None:\n",
" super().__init__()\n",
@@ -220,7 +214,7 @@
" def __init__(self, chomp_size):\n",
" super(Chomp1d, self).__init__()\n",
" self.chomp_size = chomp_size\n",
- " \n",
+ "\n",
" def forward(self, x):\n",
" return x[:, :, :-self.chomp_size].contiguous()"
]
@@ -242,7 +236,7 @@
" def __init__(self, padding, value=0.):\n",
" super().__init__(padding, value)\n",
"\n",
- " \n",
+ "\n",
"# @delegates(nn.Conv1d.__init__)\n",
"class SameConv1d(Module):\n",
" \"Conv1d with padding='same'\"\n",
@@ -296,15 +290,15 @@
" def forward(self, x):\n",
" self.padding = same_padding2d(x.shape[-2], x.shape[-1], self.ks, dilation=self.dilation) #stride=self.stride not used in padding calculation!\n",
" return self.conv2d_same(self.pad(self.padding)(x))\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"# @delegates(nn.Conv2d.__init__)\n",
"def Conv2d(ni, nf, kernel_size=None, ks=None, stride=1, padding='same', dilation=1, init='auto', bias_std=0.01, **kwargs):\n",
" \"conv1d layer with padding='same', 'valid', or any integer (defaults to 'same')\"\n",
" assert not (kernel_size and ks), 'use kernel_size or ks but not both simultaneously'\n",
" assert kernel_size is not None or ks is not None, 'you need to pass a ks'\n",
" kernel_size = kernel_size or ks\n",
- " if padding == 'same': \n",
+ " if padding == 'same':\n",
" conv = Conv2dSame(ni, nf, kernel_size, stride=stride, dilation=dilation, **kwargs)\n",
" elif padding == 'valid': conv = nn.Conv2d(ni, nf, kernel_size, stride=stride, padding=0, dilation=dilation, **kwargs)\n",
" else: conv = nn.Conv2d(ni, nf, kernel_size, stride=stride, padding=padding, dilation=dilation, **kwargs)\n",
@@ -360,8 +354,8 @@
" assert not (kernel_size and ks), 'use kernel_size or ks but not both simultaneously'\n",
" assert kernel_size is not None or ks is not None, 'you need to pass a ks'\n",
" kernel_size = kernel_size or ks\n",
- " if padding == 'same': \n",
- " if kernel_size%2==1: \n",
+ " if padding == 'same':\n",
+ " if kernel_size%2==1:\n",
" conv = nn.Conv1d(ni, nf, kernel_size, stride=stride, padding=kernel_size//2 * dilation, dilation=dilation, **kwargs)\n",
" else:\n",
" conv = SameConv1d(ni, nf, kernel_size, stride=stride, dilation=dilation, **kwargs)\n",
@@ -511,10 +505,10 @@
" self.depthwise_conv = Conv1d(ni, ni, ks, stride=stride, padding=padding, dilation=dilation, groups=ni, bias=bias)\n",
" self.pointwise_conv = nn.Conv1d(ni, nf, 1, stride=1, padding=0, dilation=1, groups=1, bias=bias)\n",
" if bias:\n",
- " if bias_std != 0: \n",
+ " if bias_std != 0:\n",
" normal_(self.depthwise_conv.bias, 0, bias_std)\n",
" normal_(self.pointwise_conv.bias, 0, bias_std)\n",
- " else: \n",
+ " else:\n",
" self.depthwise_conv.bias.data.zero_()\n",
" self.pointwise_conv.bias.data.zero_()\n",
"\n",
@@ -599,7 +593,7 @@
" if norm_type==NormType.Weight: conv = weight_norm(conv)\n",
" elif norm_type==NormType.Spectral: conv = spectral_norm(conv)\n",
" layers += [conv]\n",
- " act_bn = [] \n",
+ " act_bn = []\n",
" if act is not None: act_bn.append(act)\n",
" if bn: act_bn.append(BatchNorm(nf, norm_type=norm_type, ndim=ndim))\n",
" if inn: act_bn.append(InstanceNorm(nf, norm_type=norm_type, ndim=ndim))\n",
@@ -607,8 +601,8 @@
" if dropout: layers += [nn.Dropout(dropout)]\n",
" layers += act_bn\n",
" if xtra: layers.append(xtra)\n",
- " super().__init__(*layers) \n",
- " \n",
+ " super().__init__(*layers)\n",
+ "\n",
"Conv = named_partial('Conv', ConvBlock, norm=None, act=None)\n",
"ConvBN = named_partial('ConvBN', ConvBlock, norm='Batch', act=None)\n",
"CoordConv = named_partial('CoordConv', ConvBlock, norm=None, act=None, coord=True)\n",
@@ -662,7 +656,7 @@
" \"Squeeze and excitation module for 1d\"\n",
" nf = math.ceil(ni//reduction/8)*8\n",
" assert nf != 0, 'nf cannot be 0'\n",
- " return SequentialEx(nn.AdaptiveAvgPool1d(1), \n",
+ " return SequentialEx(nn.AdaptiveAvgPool1d(1),\n",
" ConvBlock(ni, nf, ks=1, norm=None, act=act, act_kwargs=act_kwargs),\n",
" ConvBlock(nf, ni, ks=1, norm=None, act=nn.Sigmoid), ProdLayer())"
]
@@ -738,7 +732,7 @@
" (0): AddCoords1d()\n",
" (1): Conv1d(4, 5, kernel_size=(5,), stride=(1,), padding=(2,), bias=False)\n",
" (2): BatchNorm1d(5, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
- " (3): Swish()\n",
+ " (3): SiLU()\n",
")"
]
},
@@ -842,88 +836,89 @@
" def __init__(self, dim, size, step=1): self.dim, self.size, self.step = dim, size, step\n",
" def forward(self, x:Tensor) -> Tensor: return x.unfold(dimension=self.dim, size=self.size, step=self.step)\n",
" def __repr__(self): return f\"{self.__class__.__name__}(dim={self.dim}, size={self.size}, step={self.step})\"\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class Permute(Module):\n",
" def __init__(self, *dims): self.dims = dims\n",
" def forward(self, x:Tensor) -> Tensor: return x.permute(self.dims)\n",
" def __repr__(self): return f\"{self.__class__.__name__}(dims={', '.join([str(d) for d in self.dims])})\"\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class Transpose(Module):\n",
- " def __init__(self, *dims, contiguous=False): self.dims, self.contiguous = dims, contiguous\n",
- " def forward(self, x): \n",
+ " def __init__(self, *dims, contiguous=True): self.dims, self.contiguous = dims, contiguous\n",
+ " def forward(self, x):\n",
+ " x = x.contiguous()\n",
" if self.contiguous: return x.transpose(*self.dims).contiguous()\n",
" else: return x.transpose(*self.dims)\n",
- " def __repr__(self): \n",
+ " def __repr__(self):\n",
" if self.contiguous: return f\"{self.__class__.__name__}(dims={', '.join([str(d) for d in self.dims])}).contiguous()\"\n",
" else: return f\"{self.__class__.__name__}({', '.join([str(d) for d in self.dims])})\"\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class View(Module):\n",
" def __init__(self, *shape): self.shape = shape\n",
- " def forward(self, x): \n",
+ " def forward(self, x):\n",
" return x.view(x.shape[0], -1).contiguous() if not self.shape else x.view(-1).contiguous() if self.shape == (-1,) else \\\n",
" x.view(x.shape[0], *self.shape).contiguous()\n",
" def __repr__(self): return f\"{self.__class__.__name__}({', '.join(['bs'] + [str(s) for s in self.shape])})\"\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class Reshape(Module):\n",
" def __init__(self, *shape): self.shape = shape\n",
" def forward(self, x):\n",
- " return x.reshape(x.shape[0], -1) if not self.shape else x.reshape(-1) if self.shape == (-1,) else x.reshape(x.shape[0], *self.shape)\n",
+ " return x.contiguous().reshape(x.shape[0], -1) if not self.shape else x.contiguous().reshape(-1) if self.shape == (-1,) else x.contiguous().reshape(x.shape[0], *self.shape)\n",
" def __repr__(self): return f\"{self.__class__.__name__}({', '.join(['bs'] + [str(s) for s in self.shape])})\"\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class Max(Module):\n",
" def __init__(self, dim=None, keepdim=False): self.dim, self.keepdim = dim, keepdim\n",
" def forward(self, x): return x.max(self.dim, keepdim=self.keepdim)[0]\n",
" def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim}, keepdim={self.keepdim})'\n",
"\n",
- " \n",
+ "\n",
"class LastStep(Module):\n",
" def forward(self, x): return x[..., -1]\n",
" def __repr__(self): return f'{self.__class__.__name__}()'\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class SoftMax(Module):\n",
" \"SoftMax layer\"\n",
" def __init__(self, dim=-1):\n",
" self.dim = dim\n",
" def forward(self, x):\n",
" return F.softmax(x, dim=self.dim)\n",
- " def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})' \n",
- " \n",
+ " def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'\n",
+ "\n",
"\n",
"class Clamp(Module):\n",
" def __init__(self, min=None, max=None):\n",
" self.min, self.max = min, max\n",
" def forward(self, x):\n",
" return x.clamp(min=self.min, max=self.max)\n",
- " def __repr__(self): return f'{self.__class__.__name__}(min={self.min}, max={self.max})' \n",
- " \n",
- " \n",
+ " def __repr__(self): return f'{self.__class__.__name__}(min={self.min}, max={self.max})'\n",
+ "\n",
+ "\n",
"class Clip(Module):\n",
" def __init__(self, min=None, max=None):\n",
" self.min, self.max = min, max\n",
- " \n",
+ "\n",
" def forward(self, x):\n",
" if self.min is not None:\n",
" x = torch.maximum(x, self.min)\n",
" if self.max is not None:\n",
" x = torch.minimum(x, self.max)\n",
" return x\n",
- " def __repr__(self): return f'{self.__class__.__name__}()' \n",
- " \n",
- " \n",
+ " def __repr__(self): return f'{self.__class__.__name__}()'\n",
+ "\n",
+ "\n",
"class ReZero(Module):\n",
" def __init__(self, module):\n",
" self.module = module\n",
" self.alpha = nn.Parameter(torch.zeros(1))\n",
" def forward(self, x):\n",
" return x + self.alpha * self.module(x)\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"Noop = nn.Sequential()"
]
},
@@ -935,7 +930,7 @@
{
"data": {
"text/plain": [
- "(Transpose(1, 2),\n",
+ "(Transpose(dims=1, 2).contiguous(),\n",
" Permute(dims=0, 2, 1),\n",
" View(bs, -1, 2, 10),\n",
" Transpose(dims=1, 2).contiguous(),\n",
@@ -1027,7 +1022,7 @@
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJs0lEQVR4nO3deVxU1fsH8M+AgCugoSCK+547JpJbKkmu7ZGamusvU1MpTTTXSizL5ZuaabmUuZdWLpRiahblnkuGuW+AKwyigjDn98cj4CgoAzNzZ/m8X695nXPv3OWZKzCP5557jk4ppUBERESkERetAyAiIiLnxmSEiIiINMVkhIiIiDTFZISIiIg0xWSEiIiINMVkhIiIiDTFZISIiIg0xWSEiIiINFVI6wDywmAw4OLFiyhRogR0Op3W4RAREVEeKKWQnJwMf39/uLjk3v5hF8nIxYsXERAQoHUYRERElA/nzp1D+fLlc33fLpKREiVKAJAP4+npqXE0RERElBd6vR4BAQFZ3+O5sYtkJPPWjKenJ5MRIiIiO/OoLhbswEpERESaYjJCREREmmIyQkRERJpiMkJERESaYjJCREREmmIyQkRERJpiMkJERESaYjJCREREmmIyQkRERJoyORnZsWMHunTpAn9/f+h0Oqxbt+6R+2zbtg2NGzeGh4cHqlWrhsWLF+cjVCIiInJEJicjKSkpaNCgAebMmZOn7U+dOoVOnTqhTZs2OHDgAIYPH47+/fvj559/NjlYIiIicjwmz03ToUMHdOjQIc/bz5s3D5UrV8ann34KAKhduzZ27tyJGTNmIDQ01NTTExERkYOx+ER5MTExCAkJMVoXGhqK4cOH57pPamoqUlNTs5b1er2lwiMiIjKbO3eA5GQgKUleN28CqalAWpqUt2/LKz0dUEpeBkPe66Zs+9B6WjpUkh4qKQmGRD1Ukh7hC+uhUkNvTa6bxZOR+Ph4+Pr6Gq3z9fWFXq/HrVu3UKRIkQf2iYyMxKRJkywdGhERUZ4pBVy9Chw7BsTGSpn5iosDbtyQhMM+FAJQ6u5LdN9z2HGTkfyIiIhAeHh41rJer0dAQICGERERkbNISQGOH8856bh+PW/HKFIE8PICihUD3N0BD4/sV+HCQKFCgE4HuLhIme962m3orl6Fy/Ur0F29Ct3VK3BJToQu/Q5cYIAOCjqonOvubnDx8oTO2ws6by/4V61i2Qv7EBZPRvz8/JCQkGC0LiEhAZ6enjm2igCAh4cHPDw8LB0aERE5uWvXgKgoYMMG4MQJ4Px54MKFh+9ToQJQo4bxKyAAKFEi++XmZoFgMzKkaeb4ceDHH4GNG4FDh3LetkgRoFIlwN8f8POTV+nSQNmyQK1aQNWqQKlSktXYAIsnI8HBwdi4caPRus2bNyM4ONjSpyYiInpAQgKweDHw5ZfyvZ4TH58HE44aNeQ7vGhRKwarFPD77xLsunXSEeV+1aoBDRsCjRoBDRpIoBUrSpOMnTA5Gblx4waO3/Ovd+rUKRw4cAClSpVChQoVEBERgQsXLuDrr78GALzxxhuYPXs2Ro0ahb59+2Lr1q1YtWoVNmzYYL5PQURE9BCJicCqVcD33wNbtkgjQ6Y6dYBnnwWaNpWGg+rVpdFAU7duAePGAcuXAxcvGr9XpgzQti3QsiXQtStQvrw2MZqRycnInj170KZNm6zlzL4dvXv3xuLFixEXF4ezZ89mvV+5cmVs2LABI0aMwKxZs1C+fHl8+eWXfKyXiIgsLjUVmDULmDrVuL9HUBAQFgZ06yZ3MGyGXi8Z07hxcs8IkI4m7doB4eFAq1YWugekLZ1SSmkdxKPo9Xp4eXkhKSkJnp6eWodDREQ27PZtYPNm4JtvgJ9/lu93QO5mvPaaJCA1amgbo5EjR4AVK4CffpJ6erqs9/ICxo4Fhg6VXq92KK/f3zb5NA0REZGpDAbg22+BYcOMW0HKlgUGDJDGhkK28q2XlgasXAlMnw4cOGD8XvXqQJ8+wODBgJP8B9xW/lmIiIjyxWAAVq+W2zExMbKuVCmgVy+5FdO0qTwGazNWrQJGjMjuC+LmBjzzjATbsqU8ruNkmIwQEZHdOn9eulKsXi3Lrq6yPGmSPN1qM9LT5TbM3LnSgxaQJpvBg4FBg2ygx6y2mIwQEZFdMRiAefOkT8iff8o6nQ54/XVg1CgZRsNmXL8OzJ4NfPFF9gAmOh3wf/8HzJwpnVOJyQgREdmXiROB99/PXm7cGPjwQ7nTYVOSk+Xpl8OHZbl0aaB/f2DgQBmQjLIwGSEiIrtx4gRwdxJ4dOgAzJ9vg8NsKAV89RXwwQfAmTPy7PAnnwAvvcSWkFwwGSEiIrtw6hTQpo3MhFu+PLB2rY19tysFLF0qTTcnT8q60qUl0GbNNA3N1tlS/2IiIqIcnTwJBAYC587JHY5Nm2wsEQHk9kuvXhJssWLSk/bkSSYiecCWESIismmnT8sApNevy9DtGzbYWJeLq1eB3r0lMEAGKhs9GiheXNu47AhbRoiIyGZt3SoJyOnTwGOPAWvW2FgiEhMjnVQzE5HRo6WvCBMRk7BlhIiIbNKlS0DHjjK/TLlyQHQ0ULOm1lHdlZEBjB8PREZKX5GSJWUM+sBArSOzS2wZISIimzR9uiQi7u7A/v02lIgAMsjJlCmSiLRrJwEyEck3towQEZHNOXwYmDFD6lOmyEMpNuPSJRk1FQDq1ZPetA44k641sWWEiIhsyuHDQPPmMpdckyYyjYtNWb9epgb29AQ2bmQiYgZMRoiIyGZERQFt2wJ6vfQTWbfOxia5U0pm5AOAIUNscMQ1+2RL/8REROTEFi+WUVUvX5a7H3/9JQmJTfnoI+DgQXlaZuhQraNxGExGiIhIcydPAn36SP3FF20wEbl5Exg2DIiIkOVu3WSYdzILdmAlIiLNtW0rpY+PTOtSpIi28RhZtkxm2b1xQ5affRaYO1fbmBwMW0aIiEhT69bJfHIAMHky4OWlaTjZlAKmTgV69MhOREaMAL7/HijE/8ubE68mERFpasECKUuWlAYIm3Dnjsw1s3ixLPv4yGM+vr6ahuWo2DJCRESaOXJEnqABZGR1m3hy5uxZIDhYEhEXF7klc+kSExELYssIERFpIj1dGh8MBqBLFxsZYfXoUZkMB5BbMWvWSB8RsigmI0REpImBA4E//pCnZGfP1joaALt3A6Gh2cvR0TIJHlkckxEiIrIqpeQp2UWLZHnpUqBCBW1jAgC8+SZw/bq0iBw8CNSurXVEToPJCBERWdXKlcBnnwE6HfDJJzZwF8RgAP73P2DPHlneuZOJiJXZQlchIiJyEkoB77wj9XffBcLDtY0HgDzOkzkBTufOQFCQtvE4ISYjRERkNStWABcuSN0mHuNduTJ7Bt5OnYDVq7WNx0nxNg0REVlFSopM7QIA1asDlSppGg5w/DjQvbs017RpA3z3HeDhoXFQzonJCBERWVx6utye+ftvoEQJYO1ajQO6fRto3Vr6i9SsCWzeDLi6ahyU8+JtGiIisiilgLFjgXnzZHn6dODxx7WNCYsWARcvSv3bb5mIaIzJCBERWdTo0cDHH0u9f3+gXz9t48GOHcBbb0n9o4+AwEBt4yEmI0REZDlbtmQnIp9+Kg+u6HQaBnTiBNCxo9w36tgx+yka0hSTESIisogdO4Cnn5Z616428BivUkDPntKT1s9PMiM3N42DIoDJCBERWUBiItCnj9TLlAG++krTcMTy5TIbHyCz8/n7axsPZWEyQkREZjdyJHDypIysvmUL4OOjcUCHDwMDBki9Xz+gQQNt4yEjTEaIiMisjh8HFi+W+vvvA/XqaRqO3J4ZPBi4eRNo3NhGZuWjezEZISIisxo7VvqHVqgADB+udTSQx3h37JD63LlA4cLaxkMPYDJCRERm8/ffwKpVUl+61Aa+99PSgPfek/qIEZx3xkYxGSEiIrPZtUvK4GCgZUttYwEgLSJxcfI88cSJWkdDuWAyQkREZpGWBrz5ptTbtdM2FgBARgbw8stS79YN8PTUNh7KFZMRIiIyi5kzpa+Ih0d2UqKZuDjprJqYKMuTJmkaDj0ckxEiIiqwjIzskVaHDwfKltU0HOCll4CDB6U+fTpQrZq28dBDcdZeIiIqsN9+A65elfr772sbC9atA/74Q+rbtwOtWmkaDj0aW0aIiKhAMjKAN96Qet++Go+wvnw58OKLUq9alYmInWAyQkREBTJxIhAbC5QsKZPhaSIxEYiIALp3BwwGoHp1YOtWjYIhU/E2DRER5ds//wBTp0r9gw8Ab28Ngjh+HAgNlfHnASAsDPj6a8DdXYNgKD/YMkJERPk2apQ8QRMUBAwapEEAt24BTzwhiUiJEsCMGcCyZUxE7AxbRoiIKN9iY6V86y0ZV8zqnn8++/Hd3buBmjU1CIIKii0jRESUL5s3yx0SQIb0sLp//wV+/lnqy5czEbFjTEaIiChfPvxQysqVgRo1rHzya9ekbwggM/K9+qqVAyBzYjJCREQmi4qSITwAYNo0wMXa3yazZsmgZl5eQHS0lU9O5sZkhIiITPbZZ1K2bAm88IIGAaxZIyVHV3UITEaIiMgkycnAli1Snz1bg46rS5fKM8VubhplQmRuTEaIiMgk69bJDL3VqgH16mkQwMyZUrZpo9HAJmRuTEaIiMgkixdL2bOnBq0i334L7N0r9enTrXxyshQmI0RElGdxcdmjrD//vJVPPnEi8NprUm/UCHj8cSsHQJbCZISIiPJs4kQp69Wz8i2a06eBSZOk3q0bsH69FU9OlpavZGTOnDmoVKkSChcujKCgIOzateuh28+cORM1a9ZEkSJFEBAQgBEjRuD27dv5CpiIiLRx44aMtA4AI0ZY+eSTJ0tZo4Z0YPX3t3IAZEkmJyMrV65EeHg4JkyYgH379qFBgwYIDQ3FpUuXctx+2bJlGD16NCZMmICjR4/iq6++wsqVKzFmzJgCB09ERNazZo0kJNWqAa+/bsUTp6cDixZJfeBADQY1IUsz+V90+vTpGDBgAPr06YM6depg3rx5KFq0KBYuXJjj9n/88QeaN2+O7t27o1KlSmjfvj26dev2yNYUIiKyLT/8IGWvXlbuuLp5c3a9WzcrnpisxaRkJC0tDXv37kVISEj2AVxcEBISgpiYmBz3efLJJ7F3796s5OPkyZPYuHEjOnbsmOt5UlNTodfrjV5ERKSdtDTg11+lHhpq5ZPv3Cllq1a8PeOgTJq198qVK8jIyICvr6/Rel9fX/z777857tO9e3dcuXIFLVq0gFIK6enpeOONNx56myYyMhKTMjsqERGR5saOBZKSgDJlgMBAK5749m0ZWQ3IfpKGHI7Fb7xt27YNU6ZMwdy5c7Fv3z58//332LBhA95///1c94mIiEBSUlLW69y5c5YOk4iIHiKz4+ro0YCrqxVPvHQpoNcDxYoBD2lRJ/tmUsuIj48PXF1dkZCQYLQ+ISEBfn5+Oe4zbtw49OzZE/379wcA1KtXDykpKRg4cCDGjh0Llxw6Inl4eMDDw8OU0IiIyEKuXQMuXpR6375WPvmePVIOGQKUK2flk5O1mNQy4u7ujsDAQETfM0OiwWBAdHQ0goODc9zn5s2bDyQcrnfTaqWUqfESEZGVrVsnZc2aMkmu1RgMwMaNUrfqvSGyNpNaRgAgPDwcvXv3RpMmTdC0aVPMnDkTKSkp6NOnDwCgV69eKFeuHCIjIwEAXbp0wfTp09GoUSMEBQXh+PHjGDduHLp06ZKVlBARkW06fz57TBGrd9k4fhzIvE3PWzQOzeRkJCwsDJcvX8b48eMRHx+Phg0bIioqKqtT69mzZ41aQt577z3odDq89957uHDhAkqXLo0uXbrgww8/NN+nICIii1i9WrpsVK0KjBplxRMrBbz5ptSDgqTPCDksnbKDeyV6vR5eXl5ISkqCp6en1uEQETmNp54Ctm8Hpk4F3n3Xiif+6Sega1epb9jAlhE7ldfvbw5jR0REObpzB8gcn7JLFyue+L//gLu3/vH660xEnACTESIiytGyZcCtWzK2SK1aVjzxJ58AV68Cnp7ARx9Z8cSkFSYjRET0gL17s+efefFFK04Ho5ScHAA+/VQyIXJ4TEaIiOgB8+dn163WOJGWBgwbJsmIhwfQoYOVTkxaYzJCRERGbt0CliyR+qJFQIkSVjrxV18Bn30m9WnTOMiZE2EyQkRERpYuBVJTgVKlgN69rXjizDHnn38eGDrUiicmrTEZISIiIwsXSvnuu4BOZ6WTJiZmz847ebKVTkq2gskIERFlOXoU+PNPqXfubMUTf/ONlOXLAzVqWPHEZAuYjBARUZatW7PrtWtb6aR6vfQRAYDhwwF3dyudmGwFkxEiIspy/ryUQ4ZY8RZNv34yB03ZskCvXlY6KdkSJiNERJQl8xZNQICVTrhtG7BmjdRXrwZKl7bSicmWMBkhIiIAwD//SG4AAMHBVjjhlSvASy9JPTQUaN7cCiclW8RkhIiIAADR0VIGBwMtW1rhhMuWybDvxYsDX39thROSrWIyQkREAIAzZ6S0SqvImTPAxIlSf/ttDvvu5JiMEBERAOD0aSkrVLDwiZQCevQArl+XGfiGD7fwCcnWMRkhIiIkJwObN0u9QQMLn2zWLOD33wE3N+C77wBvbwufkGwdkxEiIsK338pwHz4+Fu4v8ttvwMiRUu/TB6hTx4InI3vBZISIyMkpBXz+udSHDgVcXS10opMngVdfBdLTgbZtgblzLXQisjdMRoiInNzEicDBg/JQi8Xmp9u9G3jySeDiRbkt8803Fsx6yN4wGSEicmLp6cDUqVIfMwYoWdICJ4mLk0QkIQGoXh3Ytw/w97fAicheFdI6ACIi0s7Jk0BaGlCokMzSaxFffilZj04HxMQAjz1moRORvWLLCBGRE8sc6Kx5c8DFEt8IiYnA+PFSf/ddJiKUIyYjRERObN06KZ9+2kInuLcTytixFjoJ2TsmI0RETiouDvjlF6l36mSBE6xaJUO+A8C0adJDligHTEaIiJzUtGlSNm4MNGxo5oPr9TKOiMEAdO0qQ74T5YLJCBGRk9q1S8rQUAsc/IUXgJs3ZZTVZcuk8ypRLpiMEBE5ofR0YP9+qffsacYDGwxywMyesV9/DRQrZsYTkCNiMkJE5IROnpSGiyJFgJo1zXjgQYOApUulPnCgjLhK9AhMRoiInNCvv0pZs6YZH+ndvRuYP1/qkZHAF1+Y6cDk6JiMEBE5oTVrpDTbpHgZGdJRFZBRVi02gho5IiYjRERORilgzx6p9+5tpoN+/TUQHy/1xYvZYZVMwmSEiMjJrF8vA6MWLgzUq2eGA6anA598IvVGjWQeGiITMBkhInIyy5dL2asX4O5uhgNu3Qr8849kN998Y4YDkrNhMkJE5GQyR1197jkzHXDmTCm7dAEef9xMByVnwmSEiMiJJCcDV69KvXnzAh4sI0NGVt20SZafeaaAByRnVUjrAIiIyHp++03Kxx4DPD0LcKBbt4CXXwY2bJDlPn2A7t0LHB85J7aMEBE5iYsXs0db7dGjgAd7773sRGTWLGDhQukzQpQPTEaIiJzEmjXAtWtSHz++AAc6fx6YPl3qn3wCDB1a4NjIuTEZISJyEtu2STl8uNymybePPpLS1xd46y2OKUIFxmSEiMgJXLuWfVelwBPjHTok5ZQpMisvUQExGSEicgL//QekpQFeXjIuWYGcPCll7doFjosIYDJCROQU4uKkrFWrgHdVbt2SPiMAULlygeMiApiMEBE5hYsXpSxbtoAHmjZNJrfx95c+I0RmwGSEiMgJnDkjpb9/AQ4SGwt8+KHUP/2UHVfJbJiMEBE5gb17paxSJZ8HMBiAgQOl40mHDkBYmNliI2IyQkTk4FJSgL/+knqzZvk8yC+/ADt2AEWLAnPnslWEzIrJCBGRgxs2DLhxAyhdGggMzOdBNm+W8sUXgUqVzBUaEQAmI0REDi0xEfjqK6l/9VU+R2y/cyf7IJyVlyyAyQgRkQNbu1ZKPz+gc+d8HuSXX4CkJLk107u32WIjysRkhIjIgcXGStm+fQG6eSxZIuVbb0lWQ2RmTEaIiBzY/v1S5nvU1dOngdWrpV7gceSJcsZkhIjIgR08KGVQUD4PkNkqUr8+0LixWWIiuh+TESIiB3XuHBAfL/WaNfN5kMxWkX79+DgvWQyTESIiBzVzppQtWgClSuXjAPv3A0eOSP3ZZ80VFtEDmIwQETmoX36RctCgfOycmgp06yb1kBCgQgWzxUV0PyYjREQO6NAh4PBhubPy1FP5OEBkpDyK4+UFLF7MWzRkUflKRubMmYNKlSqhcOHCCAoKwq5dux66fWJiIgYPHoyyZcvCw8MDNWrUwMaNG/MVMBERPdrKlVJ26JCPyfH++guYMkXq8+cD5cqZNTai+xUydYeVK1ciPDwc8+bNQ1BQEGbOnInQ0FDExsaiTJkyD2yflpaGp59+GmXKlMGaNWtQrlw5nDlzBt7e3uaIn4iIcpD5FE2HDvnYsWNHGXW1a1fg5ZfNHhvR/XRKKWXKDkFBQXjiiScwe/ZsAIDBYEBAQACGDh2K0aNHP7D9vHnzMG3aNPz7779wc3PLV5B6vR5eXl5ISkqCp6dnvo5BROQslAJc7rZ779gBtGxpws4vvCDDttasKS0kXl4WiZGcQ16/v026TZOWloa9e/ciJCQk+wAuLggJCUFMTEyO+/z4448IDg7G4MGD4evri7p162LKlCnIyMjI9TypqanQ6/VGLyIiypvffsuumzQ0yJUrwIYNUl+8mIkIWY1JyciVK1eQkZEBX19fo/W+vr6Iz3yY/T4nT57EmjVrkJGRgY0bN2LcuHH49NNP8cEHH+R6nsjISHh5eWW9AgICTAmTiMip/fqrlJUqAcWKmbDjlClAWpr0Ecn3KGlEprP40zQGgwFlypTB/PnzERgYiLCwMIwdOxbz5s3LdZ+IiAgkJSVlvc6dO2fpMImIHMaWLVKOGGHCTitWADNmSH3GDD49Q1ZlUgdWHx8fuLq6IiEhwWh9QkIC/HKZPKls2bJwc3ODq6tr1rratWsjPj4eaWlpcHd3f2AfDw8PeHh4mBIaERFBGjZ27pR606Z53Gn/fqB/f6n3789Oq2R1JrWMuLu7IzAwENHR0VnrDAYDoqOjERwcnOM+zZs3x/Hjx2EwGLLWHTt2DGXLls0xESEiovzLfKTX0zOPd1oMBuCll4CUFKB1a2DuXIvGR5QTk2/ThIeHY8GCBViyZAmOHj2KQYMGISUlBX369AEA9OrVCxEREVnbDxo0CNeuXcOwYcNw7NgxbNiwAVOmTMHgwYPN9ymIiAgA8PnnUvbvn8c7Ldu2ASdPSv3bb4F8PvVIVBAmjzMSFhaGy5cvY/z48YiPj0fDhg0RFRWV1an17NmzcHHJznECAgLw888/Y8SIEahfvz7KlSuHYcOG4d133zXfpyAiIhw5AmQ+2NirVx53WrxYyi5dOLgZacbkcUa0wHFGiIgebexYeSCmc2fgp5/ysMP33wMvviiDkmzbZuKAJESPZpFxRoiIyHbFxkrZrl0eNk5Ozu602qkTExHSFJMRIiIHoBTw999Sr1kzDzvMng1cvw4UKiR9RYg0xGSEiMgBbN4MHD8ug5w9+WQedshMQCZMAEqUsGhsRI/CZISIyAFkDnTWrVseRnFfsEB6u7q4AK+9ZvHYiB6FyQgRkQPYtk3KR7aK7NgBDBwo9f79Zcx4Io0xGSEisnPx8cDu3TKuSIcOj9h4xQopdTpg+nSLx0aUF0xGiIjs3P79UtauDeQyM0e27dulXLbMxFn0iCyHyQgRkZ3L7Iv6+OOP2PDcOeCff6QeEmLRmIhMwWSEiMjOZXZebd/+IRulpQHdu0u9VSvAx8ficRHlFZMRIiI7lpEBXL4s9c6dH7Jh167Z0/lmdmAlshFMRoiI7Fhioky8CwClSuWy0d69wM8/S/3jj7NbSIhsBJMRIiI7duWKlJ6egLt7LhuNHi2lnx/wzjt5nM6XyHqYjBAR2bFz56S8O3H6gzIysgch+fRTJiJkk5iMEBHZsbVrpWzcOJcNDh0C0tOl2eSVV6wWF5EpmIwQEdmpP/4APv9c6n375rBBRgYwZozUu3aVSfGIbBCTESIiO7Vxo8zW+/TT8nrAwoXApk1SHzLEqrERmYLJCBGRnTp8WMrOnXPpCvLNN1KOHAm0bm21uIhMxWSEiMhOHTsmZa1aOby5fj3w229Sf/NNq8VElB9MRoiI7ND168DRo1KvXv2+N69eBXr3lvoLL3BmXrJ5TEaIiOzQ2LFSVqmSQ64xeTJw7ZrUBw+2ZlhE+cJkhIjIzsTHZz9FM3Lkff1F4uOB+fOlvnIl0Lat1eMjMhWTESIiO/Pll1JWrQq88cY9bxgMkp3cvg0EBgIvv6xJfESm4kPnRER2Zv16KY0ekElLk9l4//pLlocO5WirZDfYMkJEZEf+/VfyDVdXYMqUe96YOjU7EZkyBejVS5P4iPKDLSNERHZkyRIpO3S4Zz6a+Hhg0iSpT5oERERoEhtRfrFlhIjITmRkAIsWSf311++uTE8HBg6U/iK1agHvvadVeET5xmSEiMhOHDkCJCQAxYvLVDPIyJDJ7376STZ4803AhX/Wyf7wNg0RkZ34/XcpmzUD3NwArPspe9reBQtymS2PyPYxGSEisgMZGdn9RVq2hNyeyezBOmgQ0L+/ZrERFRTb84iI7MDixdkPy3RpmSiDme3eDRQunD0cK5GdYjJCRGTjlMoeVPWNN4BGv3wkk+C5uMgoq+XKaRsgUQHxNg0RkY3bsQPYtQvw8AAmjUkFgu7er/nf/+72ZCWyb2wZISKycZMnS9m9O1Bm/BtAXBxQtiz7iZDDYDJCRGTDLl0Ctm6V+siKq6TzCCD3azw8NIuLyJyYjBAR2bCdO6WsUeEWak8Mk4VWrYBx47QLisjMmIwQEdmwuDgp6ybeHWSkYUNgwwZOgkcOhckIEZENO39eSj99rFSWLJEhWIkcCJMRIiIbdvSolDVwDOjcGahfX9uAiCyAj/YSEdmo06eBn6MMAFzQyPUgsHCV1iERWQRbRoiIbNSmTcDtVBc8gV1o2dYdKF1a65CILILJCBGRjTq4WvqJPIHd0I0do3E0RJbDZISIyBb9+CN+/tUNABDc2Qdo3VrjgIgsh8kIEZENOj/pK5xCFQBAi0+e0zYYIgtjMkJEZGs++ADL9tUEAJQocgcVa3CkVXJsfJqGiMiWnD4Nw/iJmImzAIAXw9w4vhk5PLaMEBHZkn79cFWVRBz8AXDUd3IOTEaIiGzF/v3A1q2Ihx8AwMsLqFJF45iIrIDJCBGRrZg/HwDwXcAIAEDjxloGQ2Q9TEaIiGxBQgKwcCEAYFnqiwCA/v21DIjIepiMEBHZgjFjgLQ03KjbDP9d8gIAtGuncUxEVsJkhIhIa4mJwLJlAIBv2i0GAPj5cfR3ch5MRoiItJSQALRoAdy+DVSqhG1xNQAAffsCLvwLTU6CP+pERFpRCujUCThyRBYXLcaOHTKoyNNPaxkYkXUxGSEi0sqJE8DevVKfOxc/6VsjPh5wdweCgrQNjciaOAIrEZFWvvlGyurVgUGD8M3LstizJ1CkiHZhEVkbW0aIiLSQlgZ8+aXUIyKweTOwZo0s9u6tXVhEWshXMjJnzhxUqlQJhQsXRlBQEHbt2pWn/VasWAGdTofnnnsuP6clInIcR44AFy8CRYsCr7yCpUtldcOGQMuWmkZGZHUmJyMrV65EeHg4JkyYgH379qFBgwYIDQ3FpUuXHrrf6dOn8c4776Alf8uIiIAtW6Rs3hwoVgxxcbL48svahUSkFZOTkenTp2PAgAHo06cP6tSpg3nz5qFo0aJYeHfkwJxkZGSgR48emDRpEqpwogUicnYZGdn9Re62FGf+fy4wUJuQiLRkUjKSlpaGvXv3IiQkJPsALi4ICQlBTExMrvtNnjwZZcqUQb9+/fJ0ntTUVOj1eqMXEZHDmD0bOHQI8PQEXnkFSgHnz8tbZcpoGxqRFkxKRq5cuYKMjAz4+voarff19UV8fHyO++zcuRNfffUVFixYkOfzREZGwsvLK+sVEBBgSphERLbLYAC++ELqAwcCPj74/nvg6lWgeHGgWjVtwyPSgkWfpklOTkbPnj2xYMEC+Pj45Hm/iIgIJCUlZb3OnTtnwSiJiKxo1Srg6FGpj5DZeaOiZPHll4ESJTSKi0hDJo0z4uPjA1dXVyQkJBitT0hIgJ+f3wPbnzhxAqdPn0aXLl2y1hkMBjlxoUKIjY1F1apVH9jPw8MDHh4epoRGRGT7bt0CunWT+pgxgL8/AODsWVnVurVGcRFpzKSWEXd3dwQGBiI6OjprncFgQHR0NIKDgx/YvlatWjh06BAOHDiQ9eratSvatGmDAwcO8PYLETmXOXOy6337ZlWPHZOyYkUrx0NkI0wegTU8PBy9e/dGkyZN0LRpU8ycORMpKSno06cPAKBXr14oV64cIiMjUbhwYdStW9dof29vbwB4YD0RkUNLTgY++kjqQ4cCd1uFz50DTp+WSfEaN9YuPCItmZyMhIWF4fLlyxg/fjzi4+PRsGFDREVFZXVqPXv2LFw41SQRUTaDAXjySeDKFcDNDZg0KeutJUukfOIJebiGyBnplFJK6yAeRa/Xw8vLC0lJSfDkbysR2ZsffwSefVbqy5cDr76a9VaLFsDvv8sDNgMHahQfkYXk9fubTRhERJaUmAi8/bbUR40ySkRSUoDMIZratrV+aES2gskIEZGl3L4NPP88cPy4PDkzapTR2/HxcgenaFGOL0LOjckIEZGldOkCbNsGeHgA330HPPaY0duZoyTcN44kkdNhMkJEZCm7d0sZHg40a/bA2wcOSMkh4MnZMRkhIrKEpUuBpCSpjx2b4yZffill06ZWionIRjEZISIyt4wMYNw4qT//PFCs2AObnD8P/P231IcPt15oRLaIyQgRkbn984+MZObmBnz9dY6bLFkinVdbtwaqVLFueES2hskIEZG5bd8uZePGMhVvDlatkvLu4NVETo3JCBGROWVkyAhmAPDMMzluohTw339Sb97cSnER2TAmI0RE5rRkCXD4sEw206lTjptcuSIT+AIA5wslYjJCRGQ+t24BERFSHzJEJpzJwcmTUvr7yxAkRM6OyQgRkblERQGXLgGFCgETJ+a62bFjUtaoYZ2wiGwdkxEiInOZOVPKHj2AkiVz3SyzvwiTESLBZISIyBxOnwZ27JD6a6/lull6OrBmjdTr1rV8WET2gMkIEVFBJScDXbtKvWFD4Kmnct10yxbg6FHA3R145RWrREdk85iMEBEV1CefAIcOAT4+wLp10mckF999J+Vrr3GCPKJMTEaIiApCKeDzz6U+ejRQseJDN88c7OyFFywcF5EdYTJCRFQQf/0FXL4s9Zdeeuimv/8O6PWATge0aGGF2IjsBJMRIqL8Ugp45x2p9+r1yFaRlSulDAoCvLwsHBuRHWEyQkSUX7NmSXOHiwswcuQjN//jDynDwy0cF5GdYTJCRJQfJ09mj7Y6atQjn9O9dUtGiQeA+vUtHBuRnWEyQkRkKqWAN98Ebt8GnnwSmDLlkbv88guQmgqUKgVUr26FGInsCJMRIiJTZGQA3bsDP/8sg4UsXCg9Uh9h+3Ypy5eXuzpElI2/EkREpvjuO2DFCql/+ilQs+Yjdzl/XrqXALlO5Evk1JiMEBGZYsYMKYcNk5l58+CnnwCDQeqZ3UyIKBuTESKivLp0ScYVAYDhw/O829dfSzl5MlCihPnDIrJ3TEaIiPJCKWD2bCkbNgQqVcrTbocOAX/+KfWePS0WHZFdYzJCRJQXS5cC778vdRNmuPvpJynbtctz/kLkdJiMEBE9ilLAtGlS7949TwOcZdqxQ8o2bSwQF5GDYDJCRPQoU6bI/ZZixYDPPnvorLz3OnBAngAGgLZtLRcekb1jMkJE9DAzZwLvvSf1SZNk1LI8yuzr2rYtEBxs/tCIHAWTESKi3Oj18ggMANSqBYwYYdLuO3dK+YiR4omcHpMRIqKc3LwJvPACcP06UKZM9oR4Jti4Uco8jItG5NTyduOTiMiZ3LkDNG4MxMZK/5Dly026PQMAc+cC165JvVcvC8RI5EDYMkJEdC+9Xmayi42V5c8/N7n3qVLABx9IvWtXoHhxM8dI5GCYjBAR3SsyEjhzRuoffQT072/yIVatAuLi5K7O0qVmjo/IATEZISLKtGYNMHWq1IcPB0aNytdhdu2SMjCQw78T5QWTESIiAPjtt+yRVZs3lxl58+mHH6QMDzdDXEROgMkIETk3peS+SvfuUu/cGdi61eQnZzIlJgInTkg9NNR8YRI5Mj5NQ0TOrWdP4NtvpV6+PLBgAeDunu/D/fuvlOXKASVLmiE+IifAlhEicl5z5mQnIv36AUePAn5+BTpkdLSU1aoVMDYiJ8JkhIicT0aGjKw6ZIgsv/km8OWXZnkGN3MI+MceK/ChiJwGkxEicj7r1gETJki9Sxdg9myzHfqnn6Rs2dJshyRyeExGiMj5zJwpZe3awHffATqdWQ5761Z2vWNHsxySyCkwGSEi55GeDgwYIDPYubgAy5YBbm5mO3zmUzTe3jKIKxHlDZ+mISLnoJTckomKkuVPPgEaNjTrKY4fl7JqVbM1thA5BSYjROQcli/PTkRmzJARVs0s87FePklDZBrepiEix7d7NzB0qNQjIiySiNy8mT1oa1CQ2Q9P5NDYMkJEjm3vXqBFCyAtDWjSBJg40SKn+e474MoVwMdHnhQmorxjywgROa7UVKBTJ0lEypcHNm4s0OiqD/O//0nZuTPg4WGRUxA5LCYjROS4xo8HEhKkHhUFlC5tkdNcvgzs2SP1AQMscgoih8ZkhIgc040bwMcfS33kSODxxy12qq1bpfT2BoKDLXYaIofFZISIHFPmwGaAxfqJAEBcHPD++1J//XU+0kuUH0xGiMjxnDoFLFki9X79gKJFLXIagwF47TXgyBFpFeEtGqL84dM0RORYbt4E2rQBzpyRJGTMGIud6pdfsm/RbNoE1KljsVMROTS2jBCR48hsqjhzRmbg/fNPoEoVi51u0yYpO3cGmjWz2GmIHF6+kpE5c+agUqVKKFy4MIKCgrBr165ct12wYAFatmyJkiVLomTJkggJCXno9kRE+XL9OvDii8DatYCrK7BiBVCvnkVP+csvUvbta9HTEDk8k5ORlStXIjw8HBMmTMC+ffvQoEEDhIaG4tKlSzluv23bNnTr1g2//vorYmJiEBAQgPbt2+PChQsFDp6ICABw4YLMTLdunSx//bWML2JByclAbKzUmze36KmIHJ5OKaVM2SEoKAhPPPEEZs+eDQAwGAwICAjA0KFDMXr06Efun5GRgZIlS2L27Nno1atXns6p1+vh5eWFpKQkeHp6mhIuETm6xESgXTtg3z5Z/vhjeZTXwpYskadnypcHzp2z+OmI7FJev79N6sCalpaGvXv3IiIiImudi4sLQkJCEBMTk6dj3Lx5E3fu3EGpUqVy3SY1NRWpqalZy3q93pQwiciZdO0qiYinpwxsZoWBPm7dAt57T+pDhlj8dEQOz6TbNFeuXEFGRgZ8fX2N1vv6+iI+Pj5Px3j33Xfh7++PkJCQXLeJjIyEl5dX1isgIMCUMInIWUyaBPz2m9RXrLDaiGM7dgDnzwNubsDgwVY5JZFDs+rTNFOnTsWKFSuwdu1aFC5cONftIiIikJSUlPU6xzZQIrrf0KHZg5mNGQN06GC1Uy9eLGWvXvLQDhEVjEm3aXx8fODq6oqEzLke7kpISICfn99D9/3kk08wdepUbNmyBfXr13/oth4eHvDgTFNElJvYWOBuvzUMHQp8+KHVTn3unDTCAEBYmNVOS+TQTGoZcXd3R2BgIKKjo7PWGQwGREdHI/ghzaMff/wx3n//fURFRaFJkyb5j5aI6Pp1oH17qVeubDzsuxVkji1Svjzw9NNWPTWRwzJ5BNbw8HD07t0bTZo0QdOmTTFz5kykpKSgT58+AIBevXqhXLlyiIyMBAB89NFHGD9+PJYtW4ZKlSpl9S0pXrw4irN9k4hMcfw40L07cPYsUKYMEB0NuFh37MaoKCkrV7bqaYkcmsnJSFhYGC5fvozx48cjPj4eDRs2RFRUVFan1rNnz8Llnj8On3/+OdLS0vDSSy8ZHWfChAmYaMHJq4jIwaxZA/TuLcO9Fy8OrFpl9Yzg+nXgxx+lPmOGVU9N5NBMHmdECxxnhMjJXb8OBAQAKSnAU08BixYBlSpZPYy33wamTwcaNAAOHLD66YnsjkXGGSEi0sQ330giUqsWsGWLDPduZX//nd09ZcIEq5+eyKFxojwisl137gAffCBNEgAwcKAmiUhaGtCqlczD16YN8OyzVg+ByKGxZYSIbNeHH8rAZoDMxjtsmCZhDBsGZA4E/eWXVu8zS+Tw+CtFRLYpPh747DOp9+kjk99pkAWcPQt88YXUP/gAqFLF6iEQOTwmI0Rke27ckPsh164BtWsD8+cDOp3Vw1AKeOstKVu0AMaOtXoIRE6ByQgR2Zb0dODll4F//5XlH38ECmlzR3n1auCHH2QOmsmTNQmByCkwGSEi2zJ/fvbIYosWAdWqaRJGRgbwzjtSf+cdaaghIstgMkJEtuP6deD996U+dizw+uuahbJsmcxDAwB9+2oWBpFTYDJCRNozGKQVpGlT6bjq65vdLKGB6GhgyBCpt26tWeMMkdPgo71EpK0bN2TGuT//lGUfH2DzZsDbW5NwEhKAzp2B27dlbJH16zUJg8ipsGWEiLSjlAxklpmITJoE7NsH1KunWUiLFkkiUqSI9J3lfJ5ElseWESLSRkYG0K8fsHy5LM+YAQwfrmlIcXFARITUJ04EvLw0DYfIaTAZISLrO3lSeoVu3y7LkZGaJyIZGUCzZtnLAwZoFwuRs+FtGiKyntRUYMMGoG1bSUQKFwZmzQJGj9Y6MsyYIaOtAjLYa8mS2sZD5EzYMkJElqeU9Af59FPpsAoA5cpJR9XatbWNDfJE8fjxUh89GujZU9t4iJwNW0aIyLKUAkaOlGTkxg3Az0/GWN+3zyYSEUCGNLl1C6heHZgyRetoiJwPW0aIyLImT5YWEQD4/HN5esaGpr3duTN7IryXX9ZkChwip8dkhIgs49w5GTnsxx9lecIE4I03tI3pPhcvyoTABgPQvbvMyktE1sdkhIjM77ffgOefB65elVnmhg0D3ntP66iMKCW3ZW7eBEqXBubMYasIkVZsp62UiBzDhAkydOnVq0DjxsCBA8C0aZrNvJublSslEQE0HfCViMBkhIjMafVq6SMCAF26SAtJnTraxpSDr78GunWTeufOQIMG2sZD5OyYjBBRwaWlAWPGAK++KsvduklfkaJFtY3rPikpMhFw796y3KyZJCZEpC3bajclIvtz8ybw4otAVJQst28PLFigbUw5iImR8UNOnJDlsDDg228BV1dt4yIitowQUUHs3Ak89ZQkIkWLAkuWSL1YMa0jM7J+PfDkk5KIlC8P/PILsGIFExEiW8GWESIynV4vk9ytWSPLXl7Axo3yjW9DlAIWLpQnjAEgOBjYtIkT4BHZGraMEJFpEhNlUI7MRGTAAODvv20uEQGAmTOB/v2B27eBNm2khYSJCJHtYcsIEeXdzp0yTGl8vNzj+PFHoGNHraPK1caNUr76qnRUdXPTNh4iyhlbRojo0a5dk6dlWraURKRGDZtPRO7cAXbtkvrbbzMRIbJlbBkhotzduSMzx338cfYIYcHBMkqYjXVSvd/HH0vXlsceAxo10joaInoYJiNElLPt22VAjjNnZLl4cWD8eGDECJsbTfV+ffsCixZJ/d13+dQMka2z7b8oRGR9SUnA6NHAvHmyXLgw8L//SadVG28NAeTpmcxEZNw44J13tI2HiB6NyQgRievXJemYODF73UsvAbNnA76+moVlim++kSeOAcmdMkemJyLbxmSEyNldvQpMmiTNCTduyLrHHpOZ5Nq10zY2E6xYAfTqJfUuXWT8NSKyD0xGiJxRRob0CYmJAebPB86elfWVKkli8sIL0kfEDhgMwJtvAl98IcuNG8sQKDberYWI7sFfVyJnYzAAr70mTQmZAgKA998HevSwq29xpaSzamYryHPPyXgi7u6ahkVEJrKfvzpElH9KAUeOyKQsn3wCxMXJ+nr1gKFDJQmxsRl282LatOxEZO5cYNAgbeMhovxhMkLk6K5fB4YPlyaDe731FjBjBuBin2MfRkbKOGwAMHgwExEie8ZkhMhRpaYCI0cCS5dKQgIAISHAM89In5DKlbWNrwD++is7EenYEZg1S9t4iKhgmIwQOZqkJOmcOm4ccPCgrKtRQ8YO6dNH29gKSCm5yzR2rCxXrAj89JPdNu4Q0V1MRogcxc2bMtzoF1/IMO6ADD06YoQM6W7nk7PcugV06gT8+qssN2ki0+MwESGyf0xGiBzBn39Kp4kDB2S5TBkgLAwID5fHde3ckSNA+/bAxYuyPGsWMGQIExEiR8FkhMieJSRI59R7H9Ndvhx49VXNQrKE116TRKRIERmLrUsXrSMiInNiMkJkb+LigHXrpGPq7t3Zt2Sefx6YMAFo0EDT8Mztm2+yG3wOHJDuL0TkWJiMENmL/fvlEZKoKOP1jRpJr862bbWJy0IOH5aP+9NPshwRwUSEyFExGSGydZs3A+PHS7+QTPXqySR2zz4L1K8P6HTaxWcB06YBo0ZlL7/5pvH8fUTkWJiMENkipYC//5YZc7/6Sta5uABdu8oju40baxufhVy9Ko/tZs4zExgonVWbN9c2LiKyLCYjRLbCYJDWjzVrgPXrgf/+y36vXz9JQipW1C4+C9u6VcZkU0qWW7eWRiE7fyKZiPKAyQiRlpSSmXNXrpQE5OTJ7PcKFZJv5Ndfl8dJHFRKioxKP358diLy2WcyxLuD3X0iolwwGSHSQkKCzOw2fz4QH5+93tNTBtR46SWgVSugbFntYrSgO3eAY8dkkrt584DkZFnfogWwcSNQooS28RGRdTEZIbKGlBT5ll2/Hjh0SJ6MyVSkiHREfeklIDQUKF5cuzgtbMcOYOFCYNUqGVE1U+HCMnjsqFF2OXkwERUQkxEiS1BK5oXZuhXYskVeaWnG2zzxBPDOO5KIeHhoE6cVnDkDLFoE/PBD9nghgORc9esDr7wCDBwoORkROScmI0TmoBRw7hywdy+wYYO87r39AgBVqgAvvww0bSqJSECANrFawYULwMyZMibb9u3Z611cZHDYIUOAZs3YJ4SIBJMRorxSCrh0CTh+HIiNlU4Px47JUy+nTsmtmHsVKSIdUFu0kNFRa9d26G/fCxdkJPpdu4BNm4AbN7Lfa9QIGDpUJrorU0a7GInINjEZIbrX7dvyRMv+/TIE6MWL8rpwATh79sGE416FCskQoSEhQOfO0gHVgW+/3LwJ/PGHTIuzfbvkaPeqW1cmDG7UCGjY0KHzMCIqICYj5DyUAhITJak4fVoSjIQE6dRw8qS8Ll7Mfr40JzodUKGCJB2Zr+rVgapVgcqVHXZQDKXkUm3fLtPiHDggjUIGg/F2zZvLuGzBwVLnrLpElBdMRsi+ZWQA167JN+WlSw++7l1//vyDnUhzktmzskEDGWSsXDnA3x8oXx6oVAlwd7f4x9KSUtLdZcsW6QJz+LD0xb18+cFty5aVWy8vvijdYB57zPrxEpH9y1cyMmfOHEybNg3x8fFo0KABPvvsMzRt2jTX7VevXo1x48bh9OnTqF69Oj766CN07Ngx30GTg8vIkA4HyclAUpIkFPHxxq+4OODIEXnv/v+eP0rp0pJUlC8vHRgCAqRzaebLx8eh7ykoBVy5IvmZXi/lyZPAP//I6+hR4Pr1B/dzcZFGoBdekDtQDRsCfn5WD5+IHJDJycjKlSsRHh6OefPmISgoCDNnzkRoaChiY2NRJoeeaX/88Qe6deuGyMhIdO7cGcuWLcNzzz2Hffv2oW7dumb5EGQDMjLkm02vlyTi/jKv9eTkh/fLyIlOJ/8lL1PG+OXra7zs7y//lXfAfhxpadJAdPWqvC5fNm4cSkiQfrYJCXKn6lENRDodUKuWDHtSt640EtWpwzFAiMgydEo97Ab5g4KCgvDEE09g9uzZAACDwYCAgAAMHToUo0ePfmD7sLAwpKSkYP369VnrmjVrhoYNG2LevHl5Oqder4eXlxeSkpLg6elpSriOSylJAO7cAVJTH3zdvp3zekttY2oCkReFCsmIpL6+8l/w+1/Vq0s/DR8f2daOKCWDft2bg+WUl+XlpdcbDyCWV489Bnh5SVmxojzsU6eOlDVqcNwPIiq4vH5/m/QXPC0tDXv37kVERETWOhcXF4SEhCAmJibHfWJiYhAeHm60LjQ0FOvWrcv1PKmpqUhNTc1a1uv1poSZZzOf3YpTp3R3OywqKAMAKFlWAJS6+5bK6tRovKzubn53+7v7Zm+Tl3V3lw1KjmPIPL/h7jrIbQiDQfYx3H3vnhxSwfiWwv3LD65zhUIxAMVM3C8P27i4Qrm5SUfOQndLNzeoe+pwKwRVyB1wK3TPNoWgXKXM3Fe5uAD3HtsAqAsALtx3fvXw5bxuYzBIfpdZ5vYqyPuZ76Wnm3536VF0OqBUKUkufHyyG4Yyy8w7U97e0kDk4F1fiMiOmJSMXLlyBRkZGfD19TVa7+vri3///TfHfeLj43PcPv7+AaHuERkZiUmTJpkSWr6s2loaMTfqWfw8TsUAIPXui/KkeHGZi6VECWkIyqyb8vL2lhefXiEie2STbdsRERFGrSl6vR4BFhitsvdzSWh7Zocs6HSATgediy6782LWurt16O7bDjmsy943a1l3z7F0965zubsOgItOvkl0LrKfi4vRS+fqArhmLrtmr3NxAQq5yrrMY90jp36YeVmX3/3MeSxrx6DTAa6u2S8XF+Pl3F752a5QIUlCihVjAkFEZFIy4uPjA1dXVyQkJBitT0hIgF8u3er9/PxM2h4APDw84GGFTob/900Li5+DiIiIHs6k/5O5u7sjMDAQ0dHRWesMBgOio6MRHByc4z7BwcFG2wPA5s2bc92eiIiInIvJt2nCw8PRu3dvNGnSBE2bNsXMmTORkpKCPn36AAB69eqFcuXKITIyEgAwbNgwtG7dGp9++ik6deqEFStWYM+ePZg/f755PwkRERHZJZOTkbCwMFy+fBnjx49HfHw8GjZsiKioqKxOqmfPnoXLPTfBn3zySSxbtgzvvfcexowZg+rVq2PdunUcY4SIiIgA5GOcES1wnBEiIiL7k9fvb/bjJyIiIk0xGSEiIiJNMRkhIiIiTTEZISIiIk0xGSEiIiJNMRkhIiIiTTEZISIiIk0xGSEiIiJNMRkhIiIiTZk8HLwWMgeJ1ev1GkdCREREeZX5vf2owd7tIhlJTk4GAAQEBGgcCREREZkqOTkZXl5eub5vF3PTGAwGXLx4ESVKlIBOpzPbcfV6PQICAnDu3DnOefMIvFam4fXKO16rvOO1yjteq7yz5LVSSiE5ORn+/v5Gk+jezy5aRlxcXFC+fHmLHd/T05M/rHnEa2UaXq+847XKO16rvOO1yjtLXauHtYhkYgdWIiIi0hSTESIiItKUUycjHh4emDBhAjw8PLQOxebxWpmG1yvveK3yjtcq73it8s4WrpVddGAlIiIix+XULSNERESkPSYjREREpCkmI0RERKQpJiNERESkKadORubMmYNKlSqhcOHCCAoKwq5du7QOyaomTpwInU5n9KpVq1bW+7dv38bgwYPx2GOPoXjx4njxxReRkJBgdIyzZ8+iU6dOKFq0KMqUKYORI0ciPT3d2h/FInbs2IEuXbrA398fOp0O69atM3pfKYXx48ejbNmyKFKkCEJCQvDff/8ZbXPt2jX06NEDnp6e8Pb2Rr9+/XDjxg2jbQ4ePIiWLVuicOHCCAgIwMcff2zpj2Z2j7pWr7/++gM/a88884zRNs5wrSIjI/HEE0+gRIkSKFOmDJ577jnExsYabWOu37tt27ahcePG8PDwQLVq1bB48WJLfzyzy8v1euqppx742XrjjTeMtnGG6/X555+jfv36WQOXBQcHY9OmTVnv2/zPlXJSK1asUO7u7mrhwoXqyJEjasCAAcrb21slJCRoHZrVTJgwQT3++OMqLi4u63X58uWs99944w0VEBCgoqOj1Z49e1SzZs3Uk08+mfV+enq6qlu3rgoJCVH79+9XGzduVD4+PioiIkKLj2N2GzduVGPHjlXff/+9AqDWrl1r9P7UqVOVl5eXWrdunfr7779V165dVeXKldWtW7eytnnmmWdUgwYN1J9//ql+++03Va1aNdWtW7es95OSkpSvr6/q0aOHOnz4sFq+fLkqUqSI+uKLL6z1Mc3iUdeqd+/e6plnnjH6Wbt27ZrRNs5wrUJDQ9WiRYvU4cOH1YEDB1THjh1VhQoV1I0bN7K2Mcfv3cmTJ1XRokVVeHi4+ueff9Rnn32mXF1dVVRUlFU/b0Hl5Xq1bt1aDRgwwOhnKykpKet9Z7leP/74o9qwYYM6duyYio2NVWPGjFFubm7q8OHDSinb/7ly2mSkadOmavDgwVnLGRkZyt/fX0VGRmoYlXVNmDBBNWjQIMf3EhMTlZubm1q9enXWuqNHjyoAKiYmRiklX0AuLi4qPj4+a5vPP/9ceXp6qtTUVIvGbm33f8EaDAbl5+enpk2blrUuMTFReXh4qOXLlyullPrnn38UALV79+6sbTZt2qR0Op26cOGCUkqpuXPnqpIlSxpdr3fffVfVrFnTwp/IcnJLRp599tlc93HWa3Xp0iUFQG3fvl0pZb7fu1GjRqnHH3/c6FxhYWEqNDTU0h/Jou6/XkpJMjJs2LBc93Hm61WyZEn15Zdf2sXPlVPepklLS8PevXsREhKStc7FxQUhISGIiYnRMDLr+++//+Dv748qVaqgR48eOHv2LABg7969uHPnjtE1qlWrFipUqJB1jWJiYlCvXj34+vpmbRMaGgq9Xo8jR45Y94NY2alTpxAfH290fby8vBAUFGR0fby9vdGkSZOsbUJCQuDi4oK//vora5tWrVrB3d09a5vQ0FDExsbi+vXrVvo01rFt2zaUKVMGNWvWxKBBg3D16tWs95z1WiUlJQEASpUqBcB8v3cxMTFGx8jcxt7/vt1/vTJ9++238PHxQd26dREREYGbN29mveeM1ysjIwMrVqxASkoKgoOD7eLnyi4myjO3K1euICMjw+iiA4Cvry/+/fdfjaKyvqCgICxevBg1a9ZEXFwcJk2ahJYtW+Lw4cOIj4+Hu7s7vL29jfbx9fVFfHw8ACA+Pj7Ha5j5niPL/Hw5ff57r0+ZMmWM3i9UqBBKlSpltE3lypUfOEbmeyVLlrRI/Nb2zDPP4IUXXkDlypVx4sQJjBkzBh06dEBMTAxcXV2d8loZDAYMHz4czZs3R926dQHAbL93uW2j1+tx69YtFClSxBIfyaJyul4A0L17d1SsWBH+/v44ePAg3n33XcTGxuL7778H4FzX69ChQwgODsbt27dRvHhxrF27FnXq1MGBAwds/ufKKZMREh06dMiq169fH0FBQahYsSJWrVplN798ZB9effXVrHq9evVQv359VK1aFdu2bUO7du00jEw7gwcPxuHDh7Fz506tQ7ELuV2vgQMHZtXr1auHsmXLol27djhx4gSqVq1q7TA1VbNmTRw4cABJSUlYs2YNevfuje3bt2sdVp445W0aHx8fuLq6PtCTOCEhAX5+fhpFpT1vb2/UqFEDx48fh5+fH9LS0pCYmGi0zb3XyM/PL8drmPmeI8v8fA/7GfLz88OlS5eM3k9PT8e1a9ec/hpWqVIFPj4+OH78OADnu1ZDhgzB+vXr8euvv6J8+fJZ6831e5fbNp6ennb5H43crldOgoKCAMDoZ8tZrpe7uzuqVauGwMBAREZGokGDBpg1a5Zd/Fw5ZTLi7u6OwMBAREdHZ60zGAyIjo5GcHCwhpFp68aNGzhx4gTKli2LwMBAuLm5GV2j2NhYnD17NusaBQcH49ChQ0ZfIps3b4anpyfq1Klj9fitqXLlyvDz8zO6Pnq9Hn/99ZfR9UlMTMTevXuzttm6dSsMBkPWH8zg4GDs2LEDd+7cydpm8+bNqFmzpt3ddjDF+fPncfXqVZQtWxaA81wrpRSGDBmCtWvXYuvWrQ/cdjLX711wcLDRMTK3sbe/b4+6Xjk5cOAAABj9bDnL9bqfwWBAamqqffxcFbgLrJ1asWKF8vDwUIsXL1b//POPGjhwoPL29jbqSezo3n77bbVt2zZ16tQp9fvvv6uQkBDl4+OjLl26pJSSR8EqVKigtm7dqvbs2aOCg4NVcHBw1v6Zj4K1b99eHThwQEVFRanSpUs7zKO9ycnJav/+/Wr//v0KgJo+fbrav3+/OnPmjFJKHu319vZWP/zwgzp48KB69tlnc3y0t1GjRuqvv/5SO3fuVNWrVzd6XDUxMVH5+vqqnj17qsOHD6sVK1aookWL2tXjqko9/FolJyerd955R8XExKhTp06pLVu2qMaNG6vq1aur27dvZx3DGa7VoEGDlJeXl9q2bZvRo6g3b97M2sYcv3eZj2COHDlSHT16VM2ZM8fuHlVV6tHX6/jx42ry5Mlqz5496tSpU+qHH35QVapUUa1atco6hrNcr9GjR6vt27erU6dOqYMHD6rRo0crnU6nfvnlF6WU7f9cOW0yopRSn332mapQoYJyd3dXTZs2VX/++afWIVlVWFiYKlu2rHJ3d1flypVTYWFh6vjx41nv37p1S7355puqZMmSqmjRour5559XcXFxRsc4ffq06tChgypSpIjy8fFRb7/9trpz5461P4pF/PrrrwrAA6/evXsrpeTx3nHjxilfX1/l4eGh2rVrp2JjY42OcfXqVdWtWzdVvHhx5enpqfr06aOSk5ONtvn7779VixYtlIeHhypXrpyaOnWqtT6i2TzsWt28eVO1b99elS5dWrm5uamKFSuqAQMGPJD4O8O1yukaAVCLFi3K2sZcv3e//vqratiwoXJ3d1dVqlQxOoe9eNT1Onv2rGrVqpUqVaqU8vDwUNWqVVMjR440GmdEKee4Xn379lUVK1ZU7u7uqnTp0qpdu3ZZiYhStv9zpVNKqYK3rxARERHlj1P2GSEiIiLbwWSEiIiINMVkhIiIiDTFZISIiIg0xWSEiIiINMVkhIiIiDTFZISIiIg0xWSEiIiINMVkhIiIiDTFZISIiIg0xWSEiIiINMVkhIiIiDT1/+A6fWa5zfvYAAAAAElFTkSuQmCC",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAARbVJREFUeJzt3QmcTfX/x/H3GAyyZ99JKSmyRqFFKUWb/08oUimKihZRSJuWHymJNqVfC634UQql+CWiVEpK1pRd9t39Pz7n685mhlnufl/Px+M433vn3HO+c9yZ+5nv8vkm+Hw+nwAAAMIkT7guDAAAYAhGAABAWBGMAACAsCIYAQAAYUUwAgAAwopgBAAAhBXBCAAACCuCEQAAEFZ5FQUOHz6sv/76S0WKFFFCQkK4qwMAALLA8qru2LFDFSpUUJ48eaI7GLFApHLlyuGuBgAAyIE1a9aoUqVK0R2MWIuI/5spWrRouKsDAACyYPv27V5jgv9zPKqDEX/XjAUiBCMAAESX4w2xYAArAAAIK4IRAAAQVgQjAAAgrAhGAABAWBGMAACAsCIYAQAAYUUwAgAAwopgBAAAhBXBCAAAiK5g5KuvvlLbtm29RW8so9rEiROP+5pZs2apfv36SkpKUs2aNfX666/ntL4AACDeg5Fdu3apbt26GjVqVJaOX7FihS677DKdf/75WrRoke666y7dfPPN+vTTT3NSXwAAEGOyvTbNpZde6m1ZNWbMGFWvXl3Dhg3zHp922mmaM2eOnnnmGbVu3Tq7lwcAADEm6AvlzZ07V61atUrznAUh1kKSmX379nlb6lX/AACIBD6ffU5ZT4Hb9u51jzPbDh6UDh8+/nboUNaOy2yzeh2z3od90v79rsLb/pH27ZcOHHCV3LFDd409U9XqFVdMBiPr1q1T2bJl0zxnjy3A2LNnjwoWLHjUa4YOHaohQ4YEu2oAAHjsw/yXX6S//pL++UfaulVavdo9XrdO2rBB2rjRPb97tzs++iRISjqyFTvqq9d++1PsBiM50b9/f/Xt2zf5sQUulStXDmudAACxYcsW6aefpB9/dNvy5dL337tAI7vy5ZMKFJCSkjLf7JjERClPnpQt/eM82dy81yf4lGfPLuXZtkUJf/2lPH+tkdZvsA9NawfJsL4J9nxiXqlIEalIYSlffikpv/e4wsknKVyCHoyUK1dO69evT/OcPS5atGiGrSLGZt3YBgBAIPz9t2QTOV96SVq5MuNjCheWatSQSpSQihe3zy+palVrzXdb6dJSyZLSCSe4rVAhKW8o/6T3+aQ1a6SFC6V586T335f++CPjY4sVk04/XTr3XLevVUuqUMF9ExY9RZig38amTZvq448/TvPc9OnTvecBAAjW5/YPP0iTJklTpkgLFqT9erVq0plnuu2UU9xndf36IQ4usuKPPySbfWqfo3Pnumad1KyZxHoOGjaUmjWTGjWSTj1VKlVKSrBumeiQ7du+c+dOLVu2LM3UXZuyW7JkSVWpUsXrYlm7dq3eeOMN7+s9evTQ888/r/vuu0833nijPv/8c7377ruaOnVqYL8TAEDcsyDklVekJ55w3S+p2ed1u3ZSr16u9SNiv4Hff5fsM3LCBNcCkppFS7VrS02a2F/70v/9n2vSiXLZDkYWLFjg5Qzx84/t6Nq1q5fM7O+//9ZqG/VzhE3rtcCjT58+evbZZ1WpUiW98sorTOsFAASMTRD55htp0CBp9uyU56+4wgUgbdq4bpeIdPiwG7RirR/PP+9Gy6Zu+WjZ0gUe9s3UresGosSYBJ/veJOBws8GsBYrVkzbtm3zxpoAAOKbBR9z5khffmmZwV0Dgj8jhI3l6NFDGjBAOvFERaa1a6XPPnPbjBnSpk0pX7MRr82bu+DjX/+K4CgqcJ/fkdY7BgBApmza7TPPuK4Ym3abmg0yPe886cEHpTp1FHm2bZMefVT65BPp55/Tfq1IEemCC6TLLpOuu07KZIJHrCIYAQBEhbfekgYPTplAYg0GllPTejFatJBOPjlCx2wuWiSNH+++gT//dM9ZRW2w6cUXWyZQNwbEWkTiFMEIACDivfaadOONrly+vGv9sMcROEs1pR/JxoCMHGmrxaY8bwFH//7SnXe6ecLwEIwAACKW5e/q0EGaNs09tkYEm6xZpowijw3BtIp+8IH00Ucp03AtQ9lVV0lXXum+AZt2izQIRgAAEckGpdosGP9nujUoPPSQlD+/Is/mzW7OsHXHpE481rOndPvtUqVK4axdxCMYAQBEnIkT3ThOW4jOEpRZN40NTo3IEbVDh0q2Mr2tdGdTcW+6yZa4d+NBLFUrjotgBAAQMWwxOmsBefVV99hmg1r21IibortkifT009I777jxIeakk6QXX5QuvDDctYs6BCMAgIhg6dsvucStkmsszYZN4Y2oQGT/funhh6Unn5QOHnTP2dovd9wh3XJLuGsXtQhGAAAR0dthPRsWiFjSMpuIYlN2I8qOHS6BiT/LuOUFscDE1oSJyDnF0YNgBAAQ9kkoN9zgVta1Resss6otLhtx0ZIFH/5AZPRoqXt3N1MGuZYn96cAACDn+vVzq+vaLJlx4yIsELFIycaB2Eq4tn6MDUi1qbuWb55AJGBoGQEAhM3u3W4cqLFhGGefHe4apVvA7pFH3Hxi/wBVG7BqmVMRUAQjAICweeIJt7chF5aUNGJ8/bV0//0pSwB37OiabeI4ZXswEYwAAMLCcoiMGuXKlqojIsaA2tzi3r2lCRPcYxtNa5lTX3qJQCSICEYAAGFhq+9adtUaNaR77gl3bST99JPLrGaVsuRl11wjPfaYW4EPQUUwAgAIuV9+kQYOdOVu3SJgLOi+fa4FxAKRvHmlyZPdXGOEBMEIACAsrSLGZs7cdlsE5A9p315avtw9/vZbqV69MFcqvhCMAABCysaB+tO9f/ihVLJkmCvUtKn0889SUpJbFIdAJOQIRgAAIWOND5bgzNhCeOecE+YK3XuvC0SMrcZn+egRciQ9AwCExNatrhHC2Hoz9tkfthk0lkPkgQekf/87ZTqPTd9FWNAyAgAIic6dpQ0bpOLFpc8+c+NEw8YyqL78sitbPhHbEDa0jAAAgu7HH6VPPnHl55+X6tcPY2WWLk0JRGwqj03fRVgRjAAAgs4/YNVmy1oLSdhs3y717OnKBQq4Be8spwjCim4aAEDQZ86+8oor33FHGCty8KDUooX0ww9u5syXX7o9wo5wEAAQVJbI1BbEq1xZuuiiMFbEWkEsELEWkU8/lRo3DmNlkBrBCAAgaP78U5oxw5WffTaMmVZXrnTTeI2twtuyZZgqgowQjAAAguLAAemmmySfTzr7bOmqq8JYmVtvdSnfL7hAuu++MFYEGSEYAQAEnAUg117rpvDawrcjR4a5ecYqYp56KkKWB0ZqBCMAgICbOtWles+XT/rgA6lhwzBVZOdOqU4dV7b5xA0ahKkiOBaCEQBAQFlviD+HWN++Yc6w/tFH0rZtrvyf/4SxIjgWghEAQEC9955b7qVIkQgYnmE55/1TemrXDnNlkBmCEQBAQC1c6PZduoR5Rd5Jk6QvvnBlsqxGNIIRAEBALV7s9mFN+W4jaDt0cGXLuFqrVhgrg+MhGAEABHQ67zffuLJ/3GjIHToktWnjBq8YWkUiHsEIACBgpk1zE1jy55fq1g3TinzWJGMVMVdcIZUoEYaKIDsIRgAAAWPLvZjq1cOw7MusWdK557qAxJKbWIvI+++HuBLICRbKAwAExK5d0ksvufI994T44nPmSK1bS/v3u3Svb70l1agR4kogpwhGAAABMWWKW6G3QgXphhtCeOHffnNTdy0QsXTvVpGCBUNYAeQW3TQAgFyzvGL9+7vydddJefOGaMbM8OFSvXrShg0uzfvYsQQiUYhgBACQa3feKa1Y4eKAkLWKTJ8u3X23tGeP1KqVW5m3atUQXRyBRDcNACBXvvpKGjfOlcePl047LYSpXv0zZiztOwvgRS1aRgAAuTJxottb40S7diG6qHXR+FfivfVWApEoRzACAMiV7793+86dQzxodfVqNzilRYsQXhjBQDACAMhVA8VPP7nyGWeE8ML+0bKNGkknnBDCCyMYCEYAALnKM7Z5s+slCdmiuJbu/X//c+WOHUN0UQQTwQgAIEc2bXLpPYzNrg3JjFpb/Obmm91U3uLFpR49QnBRBBvBCAAgR155Rdq61WVetzxjITFokPT66658331SvnwhujCCiam9AIBsO3xYGjLElS2viGVdDbrJk6UnnnBlC0i6dg3BRREKtIwAAHLUQLF3rysPHhyCC65aJbVv78q33EIgEmMIRgAA2bJunfTvf7vybbdJZcqEoBnGEpvZeBFrgnn22SBfEKFGMAIAyHZvyb59Uvny0vPPh+CCFvH88IMrv/qqVKBACC6KUCIYAQBke+CqqVs3BIlPbQU+C0DMc89Jl1wS5AsiHAhGAABZZgnOvv3Wlfv2DcEFX3hBOnhQqllT6tUrBBdEOBCMAACy7I033L5KFemii0LQKvL44ylTdlh/JmYRjAAAspz63R+MPPpoCC5o3TM7d7rkZn36hOCCCBeCEQBAlmzZ4hKfGv8s26CxCz3yiCtbIGKZ1RCzCEYAAFny559uX7p0CFK/Dxgg/fOPm7Jz551BvhiiMhgZNWqUqlWrpgIFCqhJkyaaP3/+MY8fMWKEatWqpYIFC6py5crq06eP9vqz5QAAosKoUSnjRYLaFzR6dMoMmvHjpWLFgnhBRGUwMmHCBPXt21eDBw/Wd999p7p166p169ba4G+7S+ftt9/W/fff7x2/ZMkSvfrqq945BljUCwCIChYjTJzoyp06BfEitgie5RXxX6hFiyBdDJEkweez//2ss5aQRo0a6fkjmW4OHz7stXb07t3bCzrS69WrlxeEzJw5M/m5u+++W/PmzdOcOXOydM3t27erWLFi2rZtm4oWLZqd6gIAAmDSJOnKK6X8+e13spSUFISLWCt7kyau3L27S/PK7/yoltXP72y1jOzfv18LFy5Uq1atUk6QJ4/3eO7cuRm+plmzZt5r/F05y5cv18cff6w2bdpkep19+/Z530DqDQAQvvTvFoiYLl2CFIjs2CG1bu3KpUpJL71EIBJHsrVq76ZNm3To0CGVLVs2zfP2+Ndff83wNZ06dfJed+6558oaYQ4ePKgePXocs5tm6NChGuJfDhIAEFYjRri9xQbDhwfpIm++6QasmmHDgnQRxO1smlmzZunxxx/XCy+84I0x+fDDDzV16lQ94p+ylYH+/ft7TTr+bc2aNcGuJgAgEzNmuP1TT0lFigTpIq+9ljKN15pfEFey1TJSqlQpJSYmav369Wmet8flypXL8DUDBw7U9ddfr5ttUJKkM844Q7t27dItt9yiBx54wOvmSS8pKcnbAADhNXu2tHChK198cZAusnWr9P33rnzOOUG6CGKmZSR//vxq0KBBmsGoNoDVHjdt2jTD1+zevfuogMMCGpPNsbMAgBDr18/tr7pKql49SBexbhlbf8YSm11xRZAugphpGTE2rbdr165q2LChGjdu7OUQsZaObt26eV/v0qWLKlas6I37MG3bttXw4cN11llneTNxli1b5rWW2PP+oAQAEJkZV/1zE4KW/n3z5pSBKE8/LeXN9scSYkC2/9c7dOigjRs3atCgQVq3bp3q1aunadOmJQ9qXb16dZqWkAcffFAJCQnefu3atSpdurQXiDz22GOB/U4AAAH18stuf8opUu3aQbqIzZrZs0c6/XSpZ88gXQQxl2ckHMgzAgChZZ8M1i2zapVL93H33UG4yP797iJ//eVW4Lv++iBcBDGXZwQAEB9sPKkFIgUKSLfeGsSmFwtEbP2ZDh2CdBFEA4IRAMBRxoxxextPWrhwEC6wcqX147uy7S21K+IWwQgA4ChLlrj91VcH4eSHDrl5wpbkzLppjqR+QPwiGAEAHGXjRrcvUyYIJ//4Y+n33135ww9pFQHBCAAg82CkdOkgnPy559y+a1epXr0gXADRhmAEAJCGzbS1HCNBaRmxaTrffuvKdM/gCIIRAEAaX33l9rbKhy2gG/D88tu2SSecIDVsGOCTI1oRjAAAjhrSYZo1kxISgpRJrWNHN28YIBgBAKQ3dqzbt20bhBO/+aYrd+8e4JMjmhGMAACSTZgg7dzpyu3bB/DEn34q3XKLK19+udS4cQBPjmhHMAIA8Nig1d69XbllywAnO7vkEpdfxDKtTpoUwBMjFhCMAAA8tti6Tem1gauffBLAE8+a5fY2AOWZZ6RUi6kChncEACDNwNXhw6WCBQN44unT3f6889w6NEA6BCMAAP39t/TLL67RonXrAJ/cxouYdu0CfGLECoIRAIB+/dXta9aUSpYM4In//FNauNCVmzQJ4IkRSwhGAABascLtbd26gHr1VbcvVkxq2jTAJ0esIBgBAGjiRLevUyfAJx43zu379w/wiRFLCEYAIM799ps0ZYor+1OBBIRNzfE3uZDkDMdAMAIAcW7ECLd+neUiO+WUAJ7Y3xpSv36AB6Ig1hCMAEAc279fevddV77jjgCeePHilPEiTz4ZwBMjFhGMAECcr9C7ebNUvLh0/vkBPLE/wrF5wq1aBfDEiEUEIwAQx1avTpl1mzdvAE/8n/+4/bXXBvCkiFUEIwAQx8aPd/vatQN4UsuetnKlK198cQBPjFhFMAIAcTyLxjK1JyZKvXoF8MRDhrh9iRJShQoBPDFiFcEIAMSpN990+3PPlWrUCNBJ33lHeu89V37uuQCdFLGOYAQA4tTUqW7fqVOATrh3r9Sjh5snfOWVUufOAToxYh3BCADEoVWrpO++c+WALYz32WfS9u1SmTLS++9LCQkBOjFiHcEIAMQhfyBiM2iqVg3wdN7zznMDUYAsIhgBgDjkz9J+9dUBOqF1zUyb5sqNGwfopIgXBCMAEIe++MLtTz01QCe0QMSyp5mATs1BPCAYAYA4Y40Ys2e78mWXBSinfN++rtyggZSUFICTIp4QjABAnFmzRtq2zY0XqVcvACc8+2zp119dmXVokAMEIwAQp4NXrYsmf/4AZFv9/ntXttaRCy/Mdf0QfwhGACDOzJzp9i1aBOBk/paQCy6Qhg0LwAkRjwhGACCOLFokjRrlyhddFID+njfecOU+fXJdN8QvghEAiBM2zrRtWzeAtXr1AKxh17u321epEqCRsIhXBCMAECd+/136809XnjFDKlQoFyebPFmaNMmVH36YbKvIFYIRAIgTS5e6ff36uVwYb+dOqVs3V27XTuraNSD1Q/wiGAGAOOHPLZLr6bw2AnbLFtc9Y6v0ArlEMAIAcWLePLdv1iyXJ/IPWrUBKLnq6wEcghEAiAOjR0tz57pyo0YBmJJjrroq1/UCDMEIAMSBqVPdvk0b6cwzc3GiJUuk5ctd+eSTA1I3gGAEAGLcoUMpXTT335/Lk91+e0qSEhszAgQAwQgAxLgBA6RNm6QSJaQmTXJxIktQ4o9q7rorUNUDCEYAINa98ILb33RTLteisbEiu3dLefKwBg0CimAEAGLY1q0uLYgZODCXfT333uvKZ50lJSUFpH6AIRgBgBj25ZduX6GCVKRILk709NMuv0jevNK4cYGqHuAhGAGAGOafzmuzaHKcsd0GnAwe7MoPPSSdfnrA6gcYghEAiGFffx2ARGe29oytsmfNK7mejgMcjWAEAGKUxQ8LFuQyGHntNWnkSFdu2VJKTAxY/QA/ghEAiOEumr17pZIlpVNOycEJPv5YuvFGV65bVxo+PNBVBDwEIwAQoyZPdvtzz83BeJFJk9yKvObSS10TS7lyAa8jYAhGACAG2ZjTUaNSgpFssbnAXbu66bwWiLz/vptFAwQJ7y4AiEHTpkn79knFikk9emTzxd27S9u2SdWquRaSfPmCVEvAoWUEAGLQzz+7fadO2cwvYhlWp0xx5T59CEQQEgQjABCDpk93+2yv0Pvtt66bpmxZqXfvYFQNCEwwMmrUKFWrVk0FChRQkyZNNH/+/GMe/88//+j2229X+fLllZSUpFNOOUUf2yhtAEDArVghLVzoyv4xqFn2yy8pKd9znCUNCPKYkQkTJqhv374aM2aMF4iMGDFCrVu31tKlS1WmTJmjjt+/f78uuugi72vvv/++KlasqFWrVql48eLZvTQAIAuWLHH7005zecqybM8eacSIHDapACEMRoYPH67u3burW7du3mMLSqZOnaqxY8fq/gwy89nzW7Zs0ddff618R/oerVUFABAcq1a5fbZziwwbJv32m1S+vHTPPcGoGpD7bhpr5Vi4cKFatWqVcoI8ebzHc/0LIKQzefJkNW3a1OumKVu2rOrUqaPHH39ch2zKWCb27dun7du3p9kAAFnzzz9ub8nOsmz1aunxx1OCktKlg1I3INfByKZNm7wgwoKK1OzxunXrMnzN8uXLve4Ze52NExk4cKCGDRumRx99NNPrDB06VMWKFUveKleunJ1qAkBc8wcj2eoNt1V5rZumRQvp2muDVTUgPLNpDh8+7I0Xeemll9SgQQN16NBBDzzwgNe9k5n+/ftr27ZtyduaNWuCXU0AiO9g5Kuv3P7OOxm4isgeM1KqVCklJiZq/fr1aZ63x+UySRNsM2hsrIi9zu+0007zWlKs2yd//vxHvcZm3NgGAMh5MFKiRBZf8N570o8/uvLZZwetXkBAWkYscLDWjZkzZ6Zp+bDHNi4kI+ecc46WLVvmHef322+/eUFKRoEIACB3/vzT7TOY4Hi0tWulG25w5Wuuyeb0GyBM3TQ2rffll1/WuHHjtGTJEvXs2VO7du1Knl3TpUsXr5vFz75us2nuvPNOLwixmTc2gNUGtAIAAsvmBvz6qyvXqnWcg30+6Y47XNZV+4NywoRQVBHI/dReG/OxceNGDRo0yOtqqVevnqZNm5Y8qHX16tXeDBs/G3z66aefqk+fPjrzzDO9PCMWmPTr1y+7lwYAHIcNx9uyxY0XOW4wYsknP/zQjREZPlxK1Z0OhFKCz2ehcWSzqb02q8YGsxYtWjTc1QGAiGS/zatUcd00zz8vHbcBulkzydIy9OwpvfBCiGqJeLI9i5/frE0DADHChn9YIGINHEd6zo+9Bo0/P9Qtt4SiekCmCEYAIEYsXer2NWtKhQod48C9e6XOnV355JOlevVCUj8gMwQjABAjVq50++rVj3OgDVT9/XepYEFpypRQVA04JoIRAIgRixdnMRiZONHtO3TIwQI2QOARjABAjPDnLWvS5DijXP25oqw/B4gABCMAECNWrHD7k046xkHjx0s7drhyr14hqRdwPAQjABADDhyQVq1y5Ro1jnHgiBFuf+utUrFiIakbcDwEIwAQA6ZPt+U5JEvlkMlSYdIvv0jz57typ06hrB5wTAQjABADPvnE7du1k1IlwU47VsS/VIelZm3ePKT1A46FYAQAotzWrdLYsa7cpk0mgcgjj0iTJ0t580rjxrkU8ECEIBgBgCi3aJFb686WCPu//8vgAItUBg925UcfPc50GyD0CEYAIEbyi9Sv7xo+jvLcc27/4IMSi5QiAhGMAECUe/31lHXvMpw9YwlILEq5885QVw3IEoIRAIhiv/4qffedKx/VRWP54e+9NyXbaqlSIa8fkBUEIwAQxZ5+2u0rVcogs/ugQdLBg9KJJ0qvvhqO6gFZQjACAFFq/3635p156ql0E2Ssa+Y//0nJupqUFJY6AllBMAIAUWrSJGnXLldu3z7dF19+2e2tuaRVq5DXDcgOghEAiEKWbfXaa135ttukfPnSfdHfZOI/CIhgBCMAEIVmz3Yxhzlqtu7ChdLGjVJionT33eGoHpAtBCMAEIWmTXP7q6+WqlRJl2114EBXvvxyt1gNEOEIRgAgyuzbJ40a5cpXXJHuiyNHSp9+6kaz+qf1AhGOYAQAosyGDdKOHa583XWpvvDf/0r33+/Klv79nHPCUj8guzJKHAwAiPCF8Uzp0qlW6J03zy3Za1q2TOmqAaIALSMAEGUshYhJM1bk1lvd/swzXTdNcpQCRD7erQAQpWvRXHJJqrTvP/yQkl+EBGeIMgQjABBlXTSzZrlyly5HnnzttZSmksaNw1Y3IKcIRgAgigwZIh06JJ18stu0apXLBW9uvjnc1QNyhGAEAKLIxInp1qKx6GTvXqlMGalv33BXD8gRghEAiBJr17qGEEusmrzczDffpOQXOeGEcFYPyDGCEQCIEr/95vYnnSQVLixp/nxpyRLXRNKsWbirB+QYwQgARAmbNGOqVz/yhH8xvGuukSpVClu9gNwiGAGAKAtGqlWTtHq19Oyz7omOHcNaLyC3CEYAIArYCr0ffujKJx1c6qbS2LSaEiWk1q3DXT0gVwhGACAK2MDVxYtd+brxl0v797uumc8/Z+Aqoh5r0wBAFJgzx+3PKLFG5bcukwoVkn75RSpSJNxVA3KNlhEAiAJvvun27be+7ArvvEMggphBMAIAEe6vv6QZM1y5s96SGjVKWaEXiAF00wBAhHvySTeA9VzN1klaLo0aH+4qAQFFywgARLj//tft++gZqWxZ1zICxBCCEQCIYJ98Iq1YYb+sD+kCfS61bBnuKgEBRzACABHK1r+78UafV/6X3lXxEw5K//53uKsFBBxjRgAgQs2aJa1bl6By+luvqZs0e65UuXK4qwUEHC0jABChhg1z+3aarALnN5POOivcVQKCgmAEACKQLcabZjrv5ZeHu0pA0BCMAEAEmj/f7StpjVrknetW5gViFMEIAESgBd8c9Pbt9b6NYpWqVg13lYCgIRgBgAg0+5Md3r6x5jODBjGPYAQAIsz33x7UD6tKeOUWPeuwBg1iHsEIAESYH8bM9fbnaI4q3tc53NUBgo5gBAAiic+n8RNcsb7N5K1WLdw1AoKOYAQAIsjurxZo3q46Xrn29Q3CXR0gJAhGACCCfPHY1/pHbrzIdd0Lhrs6QEgQjABApBg3Tj9OX+cV2zf7S4ULh7tCQGiwNg0ARIJx46QbbtD7WuA9PPdfFcJdIyBkaBkBgHCbPdsLRL5Sc32nBsqTx6crrwx3pYDQIRgBgHD6+mvpssu84pflrvX255+fQMJVxJUcBSOjRo1StWrVVKBAATVp0kTz/YsoHMf48eOVkJCgKwn5AUCaMkU65xxpxw6pVCl9WqGb97Q9BcSTbAcjEyZMUN++fTV48GB99913qlu3rlq3bq0NGzYc83UrV67UPffco+bNm+emvgAQGzZulHr1cuVSpaQFC/TTMjd7hl+TiDfZDkaGDx+u7t27q1u3bqpdu7bGjBmjQoUKaezYsZm+5tChQ+rcubOGDBmiGjVq5LbOABD9bL2ZVatcIPLrr9pduqq2b3dfatw43JUDIjgY2b9/vxYuXKhWrVqlnCBPHu/x3LkufXFGHn74YZUpU0Y33XRTlq6zb98+bd++Pc0GADHj4EHpnXdc+ZFHpBNP1F9/uYeFCrEUDeJPtoKRTZs2ea0cZcuWTfO8PV63zs2NT2/OnDl69dVX9fLLL2f5OkOHDlWxYsWSt8qVK2enmgAQ2T76SFqzxgtCdN113lNffOG+VLu2lJAQ3uoBMTWbZseOHbr++uu9QKSUNUVmUf/+/bVt27bkbY390AJArPj2W7e/+mr5M5stWuSeStXwDMSNbCU9s4AiMTFR69evT/O8PS5XrtxRx//xxx/ewNW2bdsmP3f48GF34bx5tXTpUp100klHvS4pKcnbACDm7Nkjvf32UYND/PFJzZphqhcQLS0j+fPnV4MGDTRz5sw0wYU9btq06VHHn3rqqfrpp5+0aNGi5K1du3Y6//zzvTLdLwDizgMPSGvXSlWqJHfR7NyZEoxYNw0Qb7KdDt6m9Xbt2lUNGzZU48aNNWLECO3atcubXWO6dOmiihUreuM+LA9JnTpu9Um/4sWLe/v0zwNAzJs3T3ruOVceNEgqUMAr/v57yiHMpEE8ynYw0qFDB23cuFGDBg3yBq3Wq1dP06ZNSx7Uunr1am+GDQAgnWeesVwH0nnnSTfemPz0smVu36SJlJgYvuoB4ZLg8/l8inA2tddm1dhg1qJFi4a7OgCQfZs3S6VLS/Yr95NPpEsuSf7SHXdII0day7JbLw+IFVn9/KYJAwBC4Y03XCBizj8/zZc+/tjtSQOPeEUwAgDB9tVX0uDBKc0gqWYL2gRDf8IzpvUiXhGMAECwp/J27OgWw7PRqY8+mubLS5a4QyzzKhMMEa8IRgAgmGnfO3d2TR8lSkiffXZUrvc5c9z+7LOlfPnCU00g3AhGACBYnn7apX43L70kFSt21CGzZ7v9ueeGuG5ABCEYAYBg+PtvacAAVx4yRGrf/qhDbDzrjBmu3Lx5iOsHRBCCEQAIhhEjUsr+oCSdN9+05TQsu7XrpgHiFcEIAASaTZF54QVXfvFFW4wrw8Pee8/tr78+eb08IC4RjABAMLpobMEZc8MNGR6ybp00fbor33prCOsGRCCCEQAItFmz3L56ddcHk4FPP5X27pVOP11q0CC01QMiDcEIAATaK6+4fbt2x41XLr1UYjkvxDt+BAAgkCyD2ddfpwwGyYAtjGeDV83FF4ewbkCEIhgBgEB65x1p/36peHGpfv0MD7nvPpcPrWFD6aKLQl5DIOIQjABAII0Z4/b9+kkJCRkesnix2z/8cAjrBUSwjOebAQCyxzKYWZbVb791g0C6dct01u/q1a586qmhrSIQqWgZAYBAuPNOqUcPV77iCqls2QwP+/13ad8+N8mmYsXQVhGIVAQjAJDbFpGnnpJGjkzJturPZpaBcePc/pxzMp31C8QdghEAyI1nnnHjQ8ygQdJjj0mJiRkeOmmSNHToMSfaAHGJYAQActMq8vLLKd00gwcf89AHH3TlO+7IdEgJEJcIRgAgp4YNk3791a09Y5HGMbKXbdmSMovmoYdCV0UgGhCMAEBOvPuudO+9rtyzp1Sq1DEPX77c7cuXl0qUCEH9gChCMAIAOWFjQ8z550tPP33cw23GrznllCDXC4hCBCMAkF1z50o//ujKH3wgJSUd83AbL+IfuNqmTQjqB0QZghEAyI4RI6RmzVz52muz1OeydKn055+u3L59kOsHRCGCEQDIKsuw2qePK+fLJ/Xvn6WX+QORKlWkGjWCWD8gShGMAEBWbN+eEoh07izt2iWdeWaWXrp5s9tXrRrE+gFRjGAEALLCpu7u3i0VLiy9+KJrGcmi0aPdnrVogIwRjADA8ezdK73xhivfc490wglZfumBA9JXX7nyv/4VpPoBUY5gBACy0rSxbZtUpIh0333Zeunff7vZNOaCC4JTPSDaEYwAwLF8951b/M4/VqRgwWy9fOZMt69W7ZgJWoG4xo8GAGTm0CHpxhtdN81JJ0nDh2f7FOvWpZwKQMYIRgAgI9a38n//J/3wg3v81lvZbhUx/pdbowqAjBGMAEBGpk6VPvrIlR9+WGrSJNun2LPHJWg1V18d4PoBMYRgBAAy8umnbn/xxdLAgTk6xbRp0sGDUunSUqNGga0eEEsIRgAgPRvgMWWKK193Xa7X0qtfP0D1AmIUwQgApPf229LKlVKhQtIVV+ToFMuWSQsXunIWFvUF4hrBCACktmSJ1K+fK191lVS0aI5O8+abbn/ppdIZZwSwfkAMIhgBAD9bb8bGiFimsooV3cJ4uUhP4g9GABwbwQgA+D31lFtit3Jlad48102Tw1nB/mCkbt3AVhGIRQQjAGBsAZlHH3VlS25mLSM5NGuWtHatW8LmrLMCV0UgVhGMAIA1ZQwaJB0+LDVvLl1zTa5O9+yzKQvj2XI2AI6NYARAfLNxIj16SF9+KSUmSsOGSQkJOT7dhAnSpEmuTNZVIGvyZvE4AIjNfCKWR2TiRPf4mWdylZ3sjz9S0pJccol04YUBqicQ4whGAMSv++5LCURGjpR69cr1+FfLuFqzpjsdgKyhmwZAfLKsZKNHp6w9k8tAZPFiadw4Vx41ygUkALKGYARA/Jk/3+Vot5XsKlSQ7r8/16e0lpB9+1wvT6tWAaklEDcIRgDElwMHpMsuk3bscNN3J0+W8uXL1Sk3b05ZndeSt+bhNyuQLYwZARBf5s6VNm1y5e+/d0vq5lLv3i4gOe00qV273FcRiDfE7wDiywsvuH2nTgEJRCxFyRdfuPKTT+a6kQWISwQjAOLH559L777rym3bBuSUP/8srVsnFSzIVF4gp+imARAf9u9PiRZatHDpUQPAlrIxtWrleCkbIO7RMgIgPrz3nttbP8rUqQEbZbp1q9uXKBGQ0wFxiWAEQHy45x63b9NGKlw4YKf9+mu3r1IlYKcE4g7BCIDYN2CAG9hhbO2ZALHBq/68aVdfHbDTAnGHYARAbFu1Sho6NGXlupNOCtiprbfHlrexHp+mTQN2WiDuEIwAiF1r1kgNGqQ8HjMmYKc+fFi64w5X7t49ILOEgbiVo2Bk1KhRqlatmgoUKKAmTZpovqVWzsTLL7+s5s2bq0SJEt7WqlWrYx4PAAGxdKlUvbrLRla8uLRyZUDHiowdK61Y4cqPPRaw0wJxKdvByIQJE9S3b18NHjxY3333nerWravWrVtrw4YNGR4/a9YsdezYUV988YXmzp2rypUr6+KLL9batWsDUX8AyNhrr7k+FPPOO1LVqgE9/euvu71llj/xxICeGog7CT6fDcHKOmsJadSokZ5//nnv8eHDh70Ao3fv3ro/C4tNHTp0yGshsdd36dIlS9fcvn27ihUrpm3btqlo0aLZqS6AeHXppdK0aa7ZwgawBtDevVKxYi51yYwZJDsDcvv5na2Wkf3792vhwoVeV0vyCfLk8R5bq0dW7N69WwcOHFDJkiUzPWbfvn3eN5B6A4Ase/ttF4gYW0Y3wGxtPQtErPfnggsCfnog7mQrGNm0aZPXslG2bNk0z9vjdf5pc8fRr18/VahQIU1Ak97QoUO9SMq/WcsLAGTJ+++7WTPmppukY/yuyYnff5duvtmVb7tNSkgI6OmBuBTS2TRPPPGExo8fr48++sgb/JqZ/v37e006/m2NjYgHgONZskTq1s2VixRxOUUCGC34Z9Ds2OHGxg4cGLBTA3EtW2vTlCpVSomJiVq/fn2a5+1xuXLljvnaf//7314wMmPGDJ155pnHPDYpKcnbACDLbPibrTezc6d0xhnS7NluYEcA9e/ven8svnnzTekYf1MBCFbLSP78+dWgQQPNnDkz+TkbwGqPmx4j489TTz2lRx55RNOmTVPDhg2zc0kAyFqTxaOPSosXuwxkNqgjwIGIxToTJriy9QI1axbQ0wNxLdur9tq03q5du3pBRePGjTVixAjt2rVL3Y40jdoMmYoVK3rjPsyTTz6pQYMG6e233/Zyk/jHlhQuXNjbACAgTRZPPeXKfftK1aoF/BJz5rhkriecIL34YsBPD8S1bAcjHTp00MaNG70AwwKLevXqeS0e/kGtq1ev9mbY+I0ePdqbhdO+ffs057E8JQ899FAgvgcA8WzevJRA5MknpXvvDcplRoxw+4sukgoVCsolgLiV7Twj4UCeEQCZds/Ureu6Zy6/XPrvf4OWzLV2bXe5Dz5gUTwgrHlGACCiTJniAhEbUXokEWMwtGnjApGWLaWrrgraZYC4RTACIDr98ot05ZWufPvtAU/37rdvn1tvz9x9N3lFgGAgGAEQfT75RGre3E1xsQEcQ4YE7VLjx0sHDthsQtcTBCDwCEYARBdLLdCunbRli1S+vJvmcozlJXLLxoiYHj1oFQGChWAEQHSw/pJOnVx694MH3YjS336TzjoraJdcuDBlTGwW1/UEkAMEIwCiQ58+0jvvuLK1jHz8sSUsCtrlLN7xD1a1AawNGgTtUkDcIxgBEPmmT7ekRa58zz3SpElBG7Dq99xzKQNX/TlGAAQHwQiAyLZ2rdSxoytXqCA9/njQL7ltW0rs88AD0sknB/2SQFwjGAEQuayvpFcvafNm6cQTpQULpHz5gn7ZW26Rli1zE3Vuuy3olwPiHsEIgMhlU1gmTnSL302d6mbPhKhXyIwd6xpjAAQXwQiAyPTll9Krr7rywIFSkyYhy6W2dasrX3ppSC4JxD2CEQCRx3KvP/igKzduLA0aFLJL+yfsVK4ssRQWEBoEIwAiz8iRLpmZee01100TAj/9JL3wQsoCwABCI2+IrgMAWfPzzymtIqNGueRmIfD991KLFtLOnW5oStu2IbksAFpGAEQUG6RqY0MsIjj1VDetJQRsiZv27d1la9WSvvkmqPnUAKRDMAIgMvz5p9Shg7Rrl1SnjjR5spQ3NI23FnwsX56y9E2VKiG5LIAjCEYAhNf27dKYMW6NGQtEypaV5s8PaaaxAQPcvlEjqWLFkF0WwBGMGQEQHocOSc8/L/37365VxBQrJn39tVSwYMiq8fnn0qxZrhHm3XdDdlkAqdAyAiC0bGCGzZCpXl266y4XiFiq04cekpYulWrUCGl1/GNlbVG8atVCemkAR9AyAiA0DhyQhgxxq85Zd4yf5RCxTKshyq6a2mefSXPnunIIlrwBkAmCEQDBt2ePa3r49FP32FbcveIK6f77wxKE+D39tNvfcINUs2bYqgHEPYIRAMHPr37rrSlJzF55ReraNWQzZY61Bp8NTzH33hvWqgBxj2AEQPA+7fv3l4YPd+ndTzhB+ugj6aKLFAlsws7u3S7lu+UWARA+BCMAgjM+pFMn6f333WPrAxk3TmrWTJHAkpw995wrt2kjJSaGu0ZAfGM2DYDAf9LbeBB/IGKf+r/9FjGByPr1LrHrhAluyRvrQQIQXrSMAAicBQuk2293fSDGcoj07q1IYd0yTZtKK1a4xwMHSuedF+5aASAYARCY1pDrr5feess9Tkpyn/R3361IYcNWGjd2gciJJ0rDhrkqAwg/ghEAuQ9ErAXEH4h07iw98YRUqZIiyZNPugWBzauvup4kAJGBYARAzm3eLF17rTRjhntss2ciMHvYxo0p68/YGBECESCyMIAVQPatXOlGgZYqlRKIdO8uPfKIIpE11PhZ9wyAyELLCIDsGT9e6tgx5XHp0tKLL7oMqxHai/T226787LMu3QmAyEIwAiDrvv9e6tLFlStUcNlUW7d2c2QjVJ8+0rp1rty8ebhrAyAjkfsbBEDk+PtvNx7knHNcQjP7VF+9Wrr00ogORGwR4FGjXLlnT+mss8JdIwAZidzfIgDCb8kSNz23WjU38MIWvDvtNOmNNyI6balN4334YenUU11W+rp1UzKuAog8dNMAONpff0l33CF98EHKc9YqYivKtW0b0a0hZuhQafDglCEtkyaFfV0+AMfAjyeAtJYtky6/3PVxmEsucVlV7bkoMGaM9OCDrjxkiGvYYdAqENkIRgA4ixdLI0a45GV790oVK7r1Zc4+W9Hi3Xfd2BBz2WUutwgtIkDk48cUiHe//y716iV99lnKcw0bur4NmzETJay6HTq4cpMm0ocfEogA0SKyO34BBDcBx+uvS6eckhKIWJfM7NluobsoCkTefFO68kpXLlZMmjpVyp8/3LUCkFX83QDEYxAyd650zz1un3rF3QYNFG2mTXPJX03t2tLnn7uF8ABED1pGgHhx6JBrQjj5ZDczxgKRggWlfv2krVujMhAZO9alOrEhLhaIWINO2bLhrhWA7KJlBIiHhGWWKXXcOOmPP9xzBQq4lO6PPhpV3TGpffONdPPNrnz++dKUKVKhQuGuFYCcIBgBYrUr5n//k159VZowwSUrMyVKuFwhvXtLhQsrWm3YIHXq5L7NNm2k//434lOfADgGghEglvz4o5tWYtNz/XlCzJlnSn37StdcE9VBiN/LL0srVrjYavRoAhEg2hGMANFu/Xq3HO1HH0m//pryvPVZXHut1LWrdO65MfGJbWNDbIHgxx5LSWpWpUq4awUgtwhGgGhN175okeuCsUGpthiLsYDDMqVecYXUvr1UtKhigXXH/Oc/LonZ2rXuuQsukG65Jdw1AxAIBCNApH8K26DTb791i9bZ9Nvvv5fWrUt7XM2a0sCBLu1ojM1rnTPHfWuzZrnHNt7WYi1bty8pKdy1AxAIBCNApNm+XZo+3Y39mDHDzYZJz1pAbPXc00+XbrxRat1asWbHDumBB6Tnn3cxWb58buytBSY2GQhA7CAYAcJp/36XtcsGnlrQ8csvLgOq5QTxsz//69WT6tZ12VItR4gFIpZqNAbZwFRb7M5yiGza5J6zHqennpKqVw937QAEA8EIEGqWYOzrr13rx+TJ7tM3vcqVpX/9y81bbdYsbpoCbFyIZVPdt889tuBj2DDpqqvCXTMAwUQwAgS75WPNGjfN9quvXCvIDz+kPcZmvdiU26pVpWrV3Cq5lk40IUHxYudO6cknXQ4207Kl1KePGwLDYndA7OPHHAiUbduk1atdN4uNtrSBpsuXp8x0Sc0CD1uUrkUL94kbo10ux7JlizRxostK/8EHrsHI3HWXaw2JgZnIALKIYATIbsCxbJmb4WKbTa/9/XfX1fLPPxm/xrpYbLbLGWe4wOPii6XSpRUvDhyQFi50DULWQPTbb26fPk6rUUPq2dO1iBCIAPGFYARI7eBBadUq6eef3Sem5fOwbhZr8Vi5Utq48divt2m1tWq5oKNxYzfbpVy5uOpysVjNGoasxcOGxthju60ZsTX7rEvGxoRYjEaXDBCf+NFH/LA/w21REwssbLM/zS1fhz1nQYaVLYOpjfM4FlsW9qST3Hbqqa7Fw0Za2niPGEi1nl027dYmAtksZMvB9vHHGcdoFpvZ7bIJQRav2Va+fFzFaQAyQTCC2Pg0tNwcmze7gQgWWFgXigUXf/6ZEnxYC8fxAg3/VFr71LRBpJUquc3GeNgMF/tTvkgRxfOttoYjGxZj3S42LMZmJfun4BoLLmwoTMOG0oUXSnXqSBUr0vUCIMDByKhRo/T0009r3bp1qlu3rkaOHKnG9mdPJt577z0NHDhQK1eu1Mknn6wnn3xSbWzKInA8lm/DAg0b3Wh5wC2gSL/Zp6MFIVlhn4iWwtMWNLGWDAs0bPyGbWXKuCDEAo84/+S0oTGpx3fY3rpbbHiMJSNLz26XrcVnP9Y33OBiNgAIWjAyYcIE9e3bV2PGjFGTJk00YsQItW7dWkuXLlUZ+2Weztdff62OHTtq6NChuvzyy/X222/ryiuv1Hfffac69icTYjeIsGXrd+92+127XFBhn3I20NO/T13OaJ/RJ19mbIpsyZJus9GQ9ue4bRZ4+DcLRCyVZxyz/xJbW88akGyzXip7bK0b333n8q7Z48zYuI769aVGjaSzznL52KwRqWDBUH4XAGJJgs9nDa9ZZwFIo0aN9LzlaPa64Q+rcuXK6t27t+6///6jju/QoYN27dqlKVOmJD939tlnq169el5AkxXbt29XsWLFtG3bNhWNkYW/gsL+K22koE1fsM26JPzl1FtGz6d+LnUQkdN9VrpDssNmpFhgYS0Z1l1igYXt/ZuN3zjhBEXzf13q/w5L+pXZ5r/NFt/5N8vTYZs9b5uVLY6zLXXZNjt/Vti429TjO6y1wyYF2ZY/f7DvCIBYkNXP72y1jOzfv18LFy5U//79k5/LkyePWrVqpbk2dD4D9ry1pKRmLSkTLcFAJvbt2+dtqb+ZYBhx5Rcu+aWFYz6ft0v+58jeC9WOfN3/vAvfjhTSPE45zudLSPWa1I8zOCbNcamuZ48P2/6wlGrvHWODMTN6fIRPR48KTP9c5sfYZh/sJ+TyPEfkSfT+nPZZi4R9ih3Z+/LmT/PY9r58/sf5pHx2zJGyncN/botzlh3Z/M+lC6kzCrGzeozdyvSbNfTk5LmMHlu8mD4eTJ39PRQstvP3TtlmA0mtQcl6qSzrvAUfcZj6BECYZCsY2bRpkw4dOqSyNpsgFXv8q81CyICNK8noeHs+M9alM2TIEAXbuzNLae7OM4J+nbhnMdL+I9uucFcmetg4WtssTvOXbbPeKP9mjUE2gSf13jb7mo2zTb/ZMRZkWJlZLAAiRUTOprGWl9StKdYyYl1Bgdb1in90waovU34rJySk/IK2wpHNey7VMfZHf0Kqr6d5bbrnE/Kkfl2CErxxke4cR17gjkn9Ou+4I+XERLfPk0cJiXncSMEjm3eMle15O3FiohLyJh55LjH5+Yw+dNI/l5VjsvpcIM8Vjrr6b1+qWx3Qxzbmwt8gdKyNYAFAvMhWMFKqVCklJiZqfbrRbfa4nHUwZ8Cez87xJikpyduC7dY3mwf9GgAA4NiyNX8xf/78atCggWbOnJn8nA1gtcdNmzbN8DX2fOrjzfTp0zM9HgAAxJdsd9NY90nXrl3VsGFDL7eITe212TLdunXzvt6lSxdVrFjRG/dh7rzzTrVs2VLDhg3TZZddpvHjx2vBggV66aWXAv/dAACA2A9GbKruxo0bNWjQIG8Qqk3RnTZtWvIg1dWrV3szbPyaNWvm5RZ58MEHNWDAAC/pmc2kIccIAADIUZ6RcCDPCAAA0Sern9/xnfMaAACEHcEIAAAIK4IRAAAQVgQjAAAgrAhGAABAWBGMAACAsCIYAQAAYUUwAgAAwopgBAAARFc6+HDwJ4m1TG4AACA6+D+3j5fsPSqCkR07dnj7ypUrh7sqAAAgB5/jlhY+qtemOXz4sP766y8VKVJECQkJAY3YLMBZs2YNa94cB/cqe7hfWce9yjruVdZxryLjXlmIYYFIhQoV0iyiG5UtI/YNVKpUKWjnt5vPmzVruFfZw/3KOu5V1nGvso57Ff57dawWET8GsAIAgLAiGAEAAGEV18FIUlKSBg8e7O1xbNyr7OF+ZR33Kuu4V1nHvYquexUVA1gBAEDsiuuWEQAAEH4EIwAAIKwIRgAAQFgRjAAAgLCK62Bk1KhRqlatmgoUKKAmTZpo/vz5iicPPfSQl9E29Xbqqacmf33v3r26/fbbdeKJJ6pw4cK65pprtH79+jTnWL16tS677DIVKlRIZcqU0b333quDBw8qFnz11Vdq27atlznQ7s3EiRPTfN3Gfg8aNEjly5dXwYIF1apVK/3+++9pjtmyZYs6d+7sJRIqXry4brrpJu3cuTPNMT/++KOaN2/uvQ8tC+JTTz2lWLtXN9xww1HvtUsuuSTu7tXQoUPVqFEjL5u0/bxceeWVWrp0aZpjAvVzN2vWLNWvX9+bIVGzZk29/vrrijZZuV/nnXfeUe+tHj16xN39Gj16tM4888zkxGVNmzbVJ598Ej3vK1+cGj9+vC9//vy+sWPH+n7++Wdf9+7dfcWLF/etX7/eFy8GDx7sO/30031///138rZx48bkr/fo0cNXuXJl38yZM30LFizwnX322b5mzZolf/3gwYO+OnXq+Fq1auX7/vvvfR9//LGvVKlSvv79+/tigX0/DzzwgO/DDz+0GWe+jz76KM3Xn3jiCV+xYsV8EydO9P3www++du3a+apXr+7bs2dP8jGXXHKJr27dur5vvvnGN3v2bF/NmjV9HTt2TP76tm3bfGXLlvV17tzZt3jxYt8777zjK1iwoO/FF1/0xdK96tq1q3cvUr/XtmzZkuaYeLhXrVu39r322mte/RctWuRr06aNr0qVKr6dO3cG9Odu+fLlvkKFCvn69u3r++WXX3wjR470JSYm+qZNm+aLJlm5Xy1btvR+f6d+b9l7Jd7u1+TJk31Tp071/fbbb76lS5f6BgwY4MuXL59376LhfRW3wUjjxo19t99+e/LjQ4cO+SpUqOAbOnSoL56CEfvln5F//vnHeyO/9957yc8tWbLE+6CZO3eu99jerHny5PGtW7cu+ZjRo0f7ihYt6tu3b58vlqT/gD18+LCvXLlyvqeffjrNPUtKSvI+JI39sNrrvv322+RjPvnkE19CQoJv7dq13uMXXnjBV6JEiTT3q1+/fr5atWr5olVmwcgVV1yR6Wvi9V5t2LDB+76//PLLgP7c3Xfffd4fGql16NDB+3CPZunvlz8YufPOOzN9TTzfrxIlSvheeeWVqHhfxWU3zf79+7Vw4UKvWT31+jf2eO7cuYon1q1gTes1atTwmsitmc7Y/Tlw4ECae2RdOFWqVEm+R7Y/44wzVLZs2eRjWrdu7S269PPPPyuWrVixQuvWrUtzf2z9BevuS31/rLuhYcOGycfY8fZemzdvXvIxLVq0UP78+dPcQ2uK3rp1q2KJNe9a02+tWrXUs2dPbd68Oflr8Xqvtm3b5u1LliwZ0J87Oyb1OfzHRPvvt/T3y++tt95SqVKlVKdOHfXv31+7d+9O/lo83q9Dhw5p/Pjx2rVrl9ddEw3vq6hYKC/QNm3a5P1npb7pxh7/+uuvihf2wWn9ffbh8Pfff2vIkCFef/zixYu9D1r7pW8fEOnvkX3N2D6je+j/Wizzf38Zff+p7499+KaWN29e7xdp6mOqV69+1Dn8XytRooRigY0Pufrqq73v9Y8//tCAAQN06aWXer/EEhMT4/Je2Wrkd911l8455xzvQ9QE6ucus2Psg2XPnj3eGKdok9H9Mp06dVLVqlW9P6psTFG/fv28APXDDz+Mu/v1008/ecGHjQ+xcSEfffSRateurUWLFkX8+yougxE49mHgZwOfLDixH+p33303an74EB2uvfba5LL99WXvt5NOOslrLbnwwgsVj2wwoQX+c+bMCXdVovp+3XLLLWneWzag3N5TFvTaeyye1KpVyws8rAXp/fffV9euXfXll18qGsRlN40159lfY+lHEtvjcuXKKV5Z1HzKKado2bJl3n2w7qx//vkn03tk+4zuof9rscz//R3rPWT7DRs2pPm6jUy3WSPxfg+tW9B+Du29Fo/3qlevXpoyZYq++OILVapUKfn5QP3cZXaMzbKIxj80MrtfGbE/qkzq91a83K/8+fN7M1waNGjgzUSqW7eunn322ah4X8VlMGL/YfafNXPmzDRNgPbYmrjilU2jtL8m7C8Luz/58uVLc4+s6dPGlPjvke2tWTD1h8j06dO9N6Y1DcYy6y6wH8zU98eaKm18Q+r7Yz/81l/r9/nnn3vvNf8vTDvGpsVaf27qe2h/4URbt0N2/Pnnn96YEXuvxdO9svG99sFqzef2/aXvdgrUz50dk/oc/mOi7ffb8e5XRqxlwKR+b8XL/UrPfn727dsXHe8rXxxP7bWZD6+//ro3kv+WW27xpvamHkkc6+6++27frFmzfCtWrPD973//86Z02VQuG7Hunwpm0+g+//xzbypY06ZNvS39VLCLL77Ym3Zn07tKly4dM1N7d+zY4U1xs81+VIYPH+6VV61alTy1194zkyZN8v3444/ebJGMpvaeddZZvnnz5vnmzJnjO/nkk9NMV7VR7jZd9frrr/em4Nn70qbORdN01ePdK/vaPffc443at/fajBkzfPXr1/fuxd69e+PqXvXs2dObDm4/d6mnou7evTv5mED83PmnYN57773erIlRo0ZF3VTVrNyvZcuW+R5++GHvPtl7y34Wa9So4WvRokXc3a/777/fm2Vk98F+H9ljm4322WefRcX7Km6DEWNzpO0/x/KN2FRfy28QT2xKVvny5b3vv2LFit5j++H2sw/V2267zZseZm/Aq666yvtFkNrKlSt9l156qZfvwQIZC3AOHDjgiwVffPGF98GafrNpqv7pvQMHDvQ+IC2wvfDCC735/alt3rzZ+0AtXLiwN0WuW7du3odzapaj5Nxzz/XOYf8PFuTE0r2yDw77BWe/2Gx6YdWqVb28EOkD/3i4VxndI9ssl0agf+7s/6RevXrez7d9QKe+Rqzcr9WrV3uBR8mSJb33hOWmsQ/K1HlG4uV+3Xjjjd7PltXfftbs95E/EImG91WC/ZP79hUAAICcicsxIwAAIHIQjAAAgLAiGAEAAGFFMAIAAMKKYAQAAIQVwQgAAAgrghEAABBWBCMAACCsCEYAAEBYEYwAAICwIhgBAABhRTACAAAUTv8PBsg9ETckqF4AAAAASUVORK5CYII=",
"text/plain": [
""
]
@@ -1059,7 +1054,7 @@
"class Sequential(nn.Sequential):\n",
" \"\"\"Class that allows you to pass one or multiple inputs\"\"\"\n",
" def forward(self, *x):\n",
- " for i, module in enumerate(self._modules.values()): \n",
+ " for i, module in enumerate(self._modules.values()):\n",
" x = module(*x) if isinstance(x, (list, tuple, L)) else module(x)\n",
" return x"
]
@@ -1083,15 +1078,15 @@
" return self.module(x)\n",
"\n",
" # Squash samples and timesteps into a single axis\n",
- " x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size)\n",
+ " x_reshape = x.contiguous().reshape(-1, x.size(-1)) # (samples * timesteps, input_size)\n",
"\n",
" y = self.module(x_reshape)\n",
"\n",
" # We have to reshape Y\n",
" if self.batch_first:\n",
- " y = y.contiguous().view(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size)\n",
+ " y = y.contiguous().reshape(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size)\n",
" else:\n",
- " y = y.view(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size)\n",
+ " y = y.reshape(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size)\n",
"\n",
" return y"
]
@@ -1140,8 +1135,8 @@
" def forward(self, x):\n",
" if self.log_softmax: x = F.log_softmax(x, dim=-1)\n",
" return self.ms(x)\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"def get_calibrator(calibrator=None, n_classes=1, **kwargs):\n",
" if calibrator is None or not calibrator: return noop\n",
" elif calibrator.lower() == 'temp': return Temp_Scale(dirichlet=False, **kwargs)\n",
@@ -1163,7 +1158,7 @@
"c_out = 3\n",
"\n",
"t = torch.rand(bs, c_out)\n",
- "for calibrator, cal_name in zip(['temp', 'vector', 'matrix'], ['Temp_Scale', 'Vector_Scale', 'Matrix_Scale']): \n",
+ "for calibrator, cal_name in zip(['temp', 'vector', 'matrix'], ['Temp_Scale', 'Vector_Scale', 'Matrix_Scale']):\n",
" cal = get_calibrator(calibrator, n_classes=c_out)\n",
"# print(calibrator)\n",
"# print(cal.weight, cal.bias, '\\n')\n",
@@ -1246,7 +1241,7 @@
" self.class_priors = class_priors\n",
" def forward(self, x):\n",
" return x.add(self.class_priors)\n",
- " \n",
+ "\n",
"LogitAdjLayer = LogitAdjustmentLayer"
]
},
@@ -1270,22 +1265,22 @@
"source": [
"#|export\n",
"class PPV(Module):\n",
- " def __init__(self, dim=-1): \n",
+ " def __init__(self, dim=-1):\n",
" self.dim = dim\n",
- " def forward(self, x): \n",
+ " def forward(self, x):\n",
" return torch.gt(x, 0).sum(dim=self.dim).float() / x.shape[self.dim]\n",
" def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'\n",
- " \n",
+ "\n",
"\n",
"class PPAuc(Module):\n",
- " def __init__(self, dim=-1): \n",
+ " def __init__(self, dim=-1):\n",
" self.dim = dim\n",
- " def forward(self, x): \n",
+ " def forward(self, x):\n",
" x = F.relu(x).sum(self.dim) / (abs(x).sum(self.dim) + 1e-8)\n",
" return x\n",
" def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class MaxPPVPool1d(Module):\n",
" \"Drop-in replacement for AdaptiveConcatPool1d - multiplies nf by 2\"\n",
" def forward(self, x):\n",
@@ -1318,8 +1313,8 @@
"#|export\n",
"class AdaptiveWeightedAvgPool1d(Module):\n",
" '''Global Pooling layer that performs a weighted average along the temporal axis\n",
- " \n",
- " It can be considered as a channel-wise form of local temporal attention. Inspired by the paper: \n",
+ "\n",
+ " It can be considered as a channel-wise form of local temporal attention. Inspired by the paper:\n",
" Hyun, J., Seong, H., & Kim, E. (2019). Universal Pooling--A New Pooling Method for Convolutional Neural Networks. arXiv preprint arXiv:1907.11440.'''\n",
"\n",
" def __init__(self, n_in, seq_len, mult=2, n_layers=2, ln=False, dropout=0.5, act=nn.ReLU(), zero_init=True):\n",
@@ -1328,7 +1323,7 @@
" inp_mult = mult if i > 0 else 1\n",
" out_mult = mult if i < n_layers -1 else 1\n",
" p = dropout[i] if is_listy(dropout) else dropout\n",
- " layers.append(LinLnDrop(seq_len * inp_mult, seq_len * out_mult, ln=False, p=p, \n",
+ " layers.append(LinLnDrop(seq_len * inp_mult, seq_len * out_mult, ln=False, p=p,\n",
" act=act if i < n_layers-1 and n_layers > 1 else None))\n",
" self.layers = layers\n",
" self.softmax = SoftMax(-1)\n",
@@ -1355,8 +1350,8 @@
" self.flatten = Reshape()\n",
" def forward(self, x):\n",
" return self.flatten(self.gap(x))\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"class GACP1d(Module):\n",
" \"Global AdaptiveConcatPool + Flatten\"\n",
" def __init__(self, output_size=1):\n",
@@ -1364,7 +1359,7 @@
" self.flatten = Reshape()\n",
" def forward(self, x):\n",
" return self.flatten(self.gacp(x))\n",
- " \n",
+ "\n",
"\n",
"class GAWP1d(Module):\n",
" \"Global AdaptiveWeightedAvgPool1d + Flatten\"\n",
@@ -1383,13 +1378,13 @@
"source": [
"#|export\n",
"class GlobalWeightedAveragePool1d(Module):\n",
- " \"\"\" Global Weighted Average Pooling layer \n",
- " \n",
+ " \"\"\" Global Weighted Average Pooling layer\n",
+ "\n",
" Inspired by Building Efficient CNN Architecture for Offline Handwritten Chinese Character Recognition\n",
" https://arxiv.org/pdf/1804.01259.pdf\n",
" \"\"\"\n",
- " \n",
- " def __init__(self, n_in, seq_len): \n",
+ "\n",
+ " def __init__(self, n_in, seq_len):\n",
" self.weight = nn.Parameter(torch.ones(1, n_in, seq_len))\n",
" self.bias = nn.Parameter(torch.zeros(1, n_in, seq_len))\n",
"\n",
@@ -1423,20 +1418,20 @@
"#|export\n",
"class AttentionalPool1d(Module):\n",
" \"\"\"Global Adaptive Pooling layer inspired by Attentional Pooling for Action Recognition https://arxiv.org/abs/1711.01467\"\"\"\n",
- " def __init__(self, n_in, c_out, bn=False): \n",
+ " def __init__(self, n_in, c_out, bn=False):\n",
" store_attr()\n",
" self.bn = nn.BatchNorm1d(n_in) if bn else None\n",
" self.conv1 = Conv1d(n_in, 1, 1)\n",
" self.conv2 = Conv1d(n_in, c_out, 1)\n",
"\n",
" def forward(self, x):\n",
- " if self.bn is not None: x = self.bn(x) \n",
+ " if self.bn is not None: x = self.bn(x)\n",
" return (self.conv1(x) @ self.conv2(x).transpose(1,2)).transpose(1,2)\n",
- " \n",
+ "\n",
"class GAttP1d(nn.Sequential):\n",
" def __init__(self, n_in, c_out, bn=False):\n",
" super().__init__(AttentionalPool1d(n_in, c_out, bn=bn), Reshape())\n",
- " \n",
+ "\n",
"def attentional_pool_head(n_in, c_out, seq_len=None, bn=True, **kwargs):\n",
" return nn.Sequential(AttentionalPool1d(n_in, c_out, bn=bn, **kwargs), Reshape())"
]
@@ -1484,7 +1479,7 @@
"source": [
"#|export\n",
"class PoolingLayer(Module):\n",
- " def __init__(self, method='cls', seq_len=None, token=True, seq_last=True): \n",
+ " def __init__(self, method='cls', seq_len=None, token=True, seq_last=True):\n",
" method = method.lower()\n",
" assert method in ['cls', 'max', 'mean', 'max-mean', 'linear', 'conv1d', 'flatten']\n",
" if method == 'cls': assert token, 'you can only choose method=cls if a token exists'\n",
@@ -1494,11 +1489,11 @@
" if method == 'linear' or method == 'conv1d':\n",
" self.linear = nn.Linear(seq_len - token, 1)\n",
"\n",
- " def forward(self, x): \n",
+ " def forward(self, x):\n",
" if self.method == 'cls':\n",
" return x[..., 0] if self.seq_last else x[:, 0]\n",
" if self.token:\n",
- " x = x[..., 1:] if self.seq_last else x[:, 1:] \n",
+ " x = x[..., 1:] if self.seq_last else x[:, 1:]\n",
" if self.method == 'max':\n",
" return torch.max(x, -1)[0] if self.seq_last else torch.max(x, 1)[0]\n",
" elif self.method == 'mean':\n",
@@ -1510,7 +1505,7 @@
" return x.flatten(1)\n",
" elif self.method == 'linear' or self.method == 'conv1d':\n",
" return self.linear(x)[...,0] if self.seq_last else self.linear(x.transpose(1,2))[...,0]\n",
- " \n",
+ "\n",
" def __repr__(self): return f\"{self.__class__.__name__}(method={self.method}, token={self.token}, seq_last={self.seq_last})\""
]
},
@@ -1625,8 +1620,8 @@
"class RevIN(nn.Module):\n",
" \"\"\" Reversible Instance Normalization layer adapted from\n",
"\n",
- " Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September). \n",
- " Reversible instance normalization for accurate time-series forecasting against distribution shift. \n",
+ " Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).\n",
+ " Reversible instance normalization for accurate time-series forecasting against distribution shift.\n",
" In International Conference on Learning Representations.\n",
" Original code: https://github.com/ts-kim/RevIN\n",
" \"\"\"\n",
@@ -1643,20 +1638,20 @@
" if self.affine:\n",
" self.weight = nn.Parameter(torch.ones(1, c_in, 1))\n",
" self.bias = nn.Parameter(torch.zeros(1, c_in, 1))\n",
- " \n",
+ "\n",
" def forward(self, x:Tensor, mode:Tensor):\n",
" \"\"\"Args:\n",
"\n",
" x: rank 3 tensor with shape [batch size x c_in x sequence length]\n",
" mode: torch.tensor(True) to normalize data and torch.tensor(False) to reverse normalization\n",
" \"\"\"\n",
- " \n",
+ "\n",
" # Normalize\n",
" if mode: return self.normalize(x)\n",
- " \n",
+ "\n",
" # Denormalize\n",
" else: return self.denormalize(x)\n",
- " \n",
+ "\n",
" def normalize(self, x):\n",
" if self.subtract_last:\n",
" self.sub = x[..., -1].unsqueeze(-1).detach()\n",
@@ -1673,7 +1668,7 @@
" x = x.sub(self.sub)\n",
" x = x.div(self.std)\n",
" return x\n",
- " \n",
+ "\n",
" def denormalize(self, x):\n",
" if self.affine:\n",
" x = x.sub(self.bias)\n",
@@ -1697,8 +1692,8 @@
"class RevIN(nn.Module):\n",
" \"\"\" Reversible Instance Normalization layer adapted from\n",
"\n",
- " Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September). \n",
- " Reversible instance normalization for accurate time-series forecasting against distribution shift. \n",
+ " Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).\n",
+ " Reversible instance normalization for accurate time-series forecasting against distribution shift.\n",
" In International Conference on Learning Representations.\n",
" Original code: https://github.com/ts-kim/RevIN\n",
" \"\"\"\n",
@@ -1715,16 +1710,16 @@
" self.weight = nn.Parameter(torch.ones(1, c_in, 1))\n",
" self.bias = nn.Parameter(torch.zeros(1, c_in, 1))\n",
" self.sub, self.std, self.mul, self.add = torch.zeros(1), torch.ones(1), torch.ones(1), torch.zeros(1)\n",
- " \n",
+ "\n",
" def forward(self, x:Tensor, mode:Tensor):\n",
" \"\"\"Args:\n",
"\n",
" x: rank 3 tensor with shape [batch size x c_in x sequence length]\n",
" mode: torch.tensor(True) to normalize data and torch.tensor(False) to reverse normalization\n",
" \"\"\"\n",
- " \n",
+ "\n",
" # Normalize\n",
- " if mode: \n",
+ " if mode:\n",
" if self.subtract_last:\n",
" self.sub = x[..., -1].unsqueeze(-1).detach()\n",
" else:\n",
@@ -1740,9 +1735,9 @@
" x = x.sub(self.sub)\n",
" x = x.div(self.std)\n",
" return x\n",
- " \n",
+ "\n",
" # Denormalize\n",
- " else: \n",
+ " else:\n",
" if self.affine:\n",
" x = x.sub(self.bias)\n",
" x = x.div(self.weight)\n",
@@ -1976,8 +1971,8 @@
" c_out = args[1]\n",
" layers = [nn.AdaptiveAvgPool1d(adaptive_size)] if adaptive_size is not None else []\n",
" for i in range(2):\n",
- " if nf > 1: \n",
- " layers += [ConvBlock(nf, nf // 2, 1)] \n",
+ " if nf > 1:\n",
+ " layers += [ConvBlock(nf, nf // 2, 1)]\n",
" nf = nf//2\n",
" else: break\n",
" layers += [ConvBlock(nf, c_out, 1), GAP1d(1)]\n",
@@ -2198,7 +2193,7 @@
"#|export\n",
"def imputation_head(c_in, c_out, seq_len=None, ks=1, y_range=None, fc_dropout=0.):\n",
" layers = [nn.Dropout(fc_dropout), nn.Conv1d(c_in, c_out, ks)]\n",
- " if y_range is not None: \n",
+ " if y_range is not None:\n",
" y_range = (tensor(y_range[0]), tensor(y_range[1]))\n",
" layers += [SigmoidRange(*y_range)]\n",
" return nn.Sequential(*layers)"
@@ -2267,10 +2262,10 @@
" fd *= _d\n",
" shape.append(_d)\n",
" if n_out > 1: shape.append(n_out)\n",
- " else: \n",
+ " else:\n",
" fd = d\n",
" shape = [d, n_out] if n_out > 1 else [d]\n",
- " \n",
+ "\n",
" conv = [BatchNorm(n_in, ndim=1)] if conv_bn else []\n",
" conv.append(Conv1d(n_in, n_out, 1, padding=0, bias=not conv_bn, **kwargs))\n",
" l = [Transpose(-1, -2), BatchNorm(seq_len, ndim=1), Transpose(-1, -2)] if lin_bn else []\n",
@@ -2282,7 +2277,7 @@
" layers += [Reshape(*shape)]\n",
"\n",
" super().__init__(*layers)\n",
- " \n",
+ "\n",
"conv_lin_nd_head = create_conv_lin_nd_head\n",
"conv_lin_3d_head = create_conv_lin_nd_head # included for compatibility\n",
"create_conv_lin_3d_head = create_conv_lin_nd_head # included for compatibility"
@@ -2296,12 +2291,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(1.7074, grad_fn=),\n",
+ "(TensorBase(1.7974, grad_fn=),\n",
" create_conv_lin_nd_head(\n",
" (0): Conv1d(32, 5, kernel_size=(1,), stride=(1,))\n",
" (1): Dropout(p=0.5, inplace=False)\n",
" (2): Linear(in_features=10, out_features=2, bias=True)\n",
- " (3): Transpose(-1, -2)\n",
+ " (3): Transpose(dims=-1, -2).contiguous()\n",
" (4): Reshape(bs, 2, 5)\n",
" ))"
]
@@ -2334,12 +2329,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(1.6561, grad_fn=),\n",
+ "(TensorBase(1.7111, grad_fn=),\n",
" create_conv_lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=10, out_features=16, bias=True)\n",
" (2): Conv1d(32, 5, kernel_size=(1,), stride=(1,))\n",
- " (3): Transpose(-1, -2)\n",
+ " (3): Transpose(dims=-1, -2).contiguous()\n",
" (4): Reshape(bs, 2, 8, 5)\n",
" ))"
]
@@ -2372,12 +2367,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(0.6017, grad_fn=),\n",
+ "(TensorBase(0.5055, grad_fn=),\n",
" create_conv_lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=10, out_features=2, bias=True)\n",
" (2): Conv1d(32, 1, kernel_size=(1,), stride=(1,))\n",
- " (3): Transpose(-1, -2)\n",
+ " (3): Transpose(dims=-1, -2).contiguous()\n",
" (4): Reshape(bs, 2)\n",
" ))"
]
@@ -2410,12 +2405,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(0.5439, grad_fn=),\n",
+ "(TensorBase(0.6870, grad_fn=),\n",
" create_conv_lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Linear(in_features=10, out_features=6, bias=True)\n",
" (2): Conv1d(32, 1, kernel_size=(1,), stride=(1,))\n",
- " (3): Transpose(-1, -2)\n",
+ " (3): Transpose(dims=-1, -2).contiguous()\n",
" (4): Reshape(bs, 2, 3)\n",
" ))"
]
@@ -2464,10 +2459,10 @@
" fd *= _d\n",
" shape.append(_d)\n",
" if n_out > 1: shape.append(n_out)\n",
- " else: \n",
+ " else:\n",
" fd = d\n",
" shape = [d, n_out] if n_out > 1 else [d]\n",
- " \n",
+ "\n",
" layers = []\n",
" if use_bn:\n",
" layers += [nn.BatchNorm1d(n_in)]\n",
@@ -2492,7 +2487,7 @@
" layers += [Reshape(*shape)]\n",
"\n",
" super().__init__(*layers)\n",
- " \n",
+ "\n",
"create_lin_nd_head = lin_nd_head\n",
"lin_3d_head = lin_nd_head # included for backwards compatiblity\n",
"create_lin_3d_head = lin_nd_head # included for backwards compatiblity"
@@ -2527,7 +2522,7 @@
{
"data": {
"text/plain": [
- "(TensorBase(1.8360, grad_fn=),\n",
+ "(TensorBase(1.8153, grad_fn=),\n",
" lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Reshape(bs)\n",
@@ -2564,7 +2559,7 @@
{
"data": {
"text/plain": [
- "(TensorBase(1.7557, grad_fn=),\n",
+ "(TensorBase(1.8502, grad_fn=),\n",
" lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Reshape(bs)\n",
@@ -2601,7 +2596,7 @@
{
"data": {
"text/plain": [
- "(TensorBase(0.5978, grad_fn=),\n",
+ "(TensorBase(0.8992, grad_fn=),\n",
" lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Reshape(bs)\n",
@@ -2638,7 +2633,7 @@
{
"data": {
"text/plain": [
- "(TensorBase(0.8286, grad_fn=),\n",
+ "(TensorBase(0.8099, grad_fn=),\n",
" lin_nd_head(\n",
" (0): Dropout(p=0.5, inplace=False)\n",
" (1): Reshape(bs)\n",
@@ -2689,7 +2684,7 @@
" fd *= _d\n",
" shape.append(_d)\n",
" if n_out > 1: shape.append(n_out)\n",
- " else: \n",
+ " else:\n",
" fd = d\n",
" shape = [d, n_out] if n_out > 1 else [d]\n",
"\n",
@@ -2752,7 +2747,7 @@
" fd *= _d\n",
" shape.append(_d)\n",
" if n_out > 1: shape.append(n_out)\n",
- " else: \n",
+ " else:\n",
" fd = d\n",
" shape = [d, n_out] if n_out > 1 else [d]\n",
"\n",
@@ -2809,7 +2804,7 @@
" layers += [Conv(n_in, n_out, 1, **kwargs), Transpose(-1,-2)]\n",
" if n_out == 1: layers += [Squeeze(-1)]\n",
" super().__init__(*layers)\n",
- " \n",
+ "\n",
"conv_3d_head = create_conv_3d_head"
]
},
@@ -2821,12 +2816,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(1.7321, grad_fn=),\n",
+ "(TensorBase(1.6665, grad_fn=),\n",
" create_conv_3d_head(\n",
" (0): ConvBlock(\n",
" (0): Conv1d(32, 5, kernel_size=(1,), stride=(1,))\n",
" )\n",
- " (1): Transpose(-1, -2)\n",
+ " (1): Transpose(dims=-1, -2).contiguous()\n",
" ))"
]
},
@@ -2858,12 +2853,12 @@
{
"data": {
"text/plain": [
- "(TensorBase(0.5833, grad_fn=),\n",
+ "(TensorBase(0.6966, grad_fn=),\n",
" create_conv_3d_head(\n",
" (0): ConvBlock(\n",
" (0): Conv1d(32, 1, kernel_size=(1,), stride=(1,))\n",
" )\n",
- " (1): Transpose(-1, -2)\n",
+ " (1): Transpose(dims=-1, -2).contiguous()\n",
" (2): Squeeze(dim=-1)\n",
" ))"
]
@@ -2897,7 +2892,7 @@
"#|export\n",
"def universal_pool_head(n_in, c_out, seq_len, mult=2, pool_n_layers=2, pool_ln=True, pool_dropout=0.5, pool_act=nn.ReLU(),\n",
" zero_init=True, bn=True, fc_dropout=0.):\n",
- " return nn.Sequential(AdaptiveWeightedAvgPool1d(n_in, seq_len, n_layers=pool_n_layers, mult=mult, ln=pool_ln, dropout=pool_dropout, act=pool_act), \n",
+ " return nn.Sequential(AdaptiveWeightedAvgPool1d(n_in, seq_len, n_layers=pool_n_layers, mult=mult, ln=pool_ln, dropout=pool_dropout, act=pool_act),\n",
" Reshape(), LinBnDrop(n_in, c_out, p=fc_dropout, bn=bn))"
]
},
@@ -2923,7 +2918,7 @@
"outputs": [],
"source": [
"#|export\n",
- "heads = [mlp_head, fc_head, average_pool_head, max_pool_head, concat_pool_head, pool_plus_head, conv_head, rnn_head, \n",
+ "heads = [mlp_head, fc_head, average_pool_head, max_pool_head, concat_pool_head, pool_plus_head, conv_head, rnn_head,\n",
" conv_lin_nd_head, lin_nd_head, conv_3d_head, attentional_pool_head, universal_pool_head, gwa_pool_head]"
]
},
@@ -2958,15 +2953,15 @@
"c_out = 14\n",
"d = 5\n",
"t = torch.rand(bs, c_in, seq_len)\n",
- "for head in heads: \n",
+ "for head in heads:\n",
" print(head.__name__)\n",
" if head.__name__ == \"create_conv_3d_head\":\n",
" h = head(c_in, c_out, seq_len, seq_len)\n",
" test_eq(h(t).shape, (bs, seq_len, c_out))\n",
- " elif 'nd' in head.__name__: \n",
+ " elif 'nd' in head.__name__:\n",
" h = head(c_in, c_out, seq_len, d)\n",
" test_eq(h(t).shape, (bs, d, c_out))\n",
- " else: \n",
+ " else:\n",
" h = head(c_in, c_out, seq_len)\n",
" test_eq(h(t).shape, (bs, c_out))"
]
@@ -3031,7 +3026,7 @@
" scale = self.sigma * (x.detach() if self.is_relative_detach else x)\n",
" sampled_noise = torch.empty(x.size(), device=x.device).normal_() * scale\n",
" x = x + sampled_noise\n",
- " return x "
+ " return x"
]
},
{
@@ -3167,11 +3162,11 @@
"source": [
"#|export\n",
"class ScaledDotProductAttention(Module):\n",
- " r\"\"\"Scaled Dot-Product Attention module (Attention is all you need by Vaswani et al., 2017) with optional residual attention from previous layer \n",
- " (Realformer: Transformer likes residual attention by He et al, 2020) and locality self sttention (Vision Transformer for Small-Size Datasets \n",
+ " r\"\"\"Scaled Dot-Product Attention module (Attention is all you need by Vaswani et al., 2017) with optional residual attention from previous layer\n",
+ " (Realformer: Transformer likes residual attention by He et al, 2020) and locality self sttention (Vision Transformer for Small-Size Datasets\n",
" by Lee et al, 2021)\"\"\"\n",
"\n",
- " def __init__(self, d_model, n_heads, attn_dropout=0., res_attention=False, lsa=False): \n",
+ " def __init__(self, d_model, n_heads, attn_dropout=0., res_attention=False, lsa=False):\n",
" self.attn_dropout = nn.Dropout(attn_dropout)\n",
" self.res_attention = res_attention\n",
" head_dim = d_model // n_heads\n",
@@ -3188,7 +3183,7 @@
" key_padding_mask: [bs x seq_len]\n",
" attn_mask : [1 x seq_len x seq_len]\n",
"\n",
- " Output shape: \n",
+ " Output shape:\n",
" output: [bs x n_heads x q_len x d_v]\n",
" attn : [bs x n_heads x q_len x seq_len]\n",
" scores : [bs x n_heads x q_len x seq_len]\n",
@@ -3198,7 +3193,7 @@
" attn_scores = torch.matmul(q, k) * self.scale # attn_scores : [bs x n_heads x max_q_len x q_len]\n",
"\n",
" # Add pre-softmax attention scores from the previous layer (optional)\n",
- " if prev is not None: attn_scores = attn_scores + prev \n",
+ " if prev is not None: attn_scores = attn_scores + prev\n",
"\n",
" # Attention mask (optional)\n",
" if attn_mask is not None: # attn_mask with shape [q_len x seq_len] - only used when q_len == seq_len\n",
@@ -3230,8 +3225,8 @@
{
"data": {
"text/plain": [
- "(tensor(1.3535e-10, grad_fn=),\n",
- " tensor(1.0555, grad_fn=))"
+ "(tensor(-7.5748e-11, grad_fn=),\n",
+ " tensor(1.0723, grad_fn=))"
]
},
"execution_count": null,
@@ -3324,9 +3319,9 @@
" if V is None: V = Q\n",
"\n",
" # Linear (+ split in multiple heads)\n",
- " q_s = self.W_Q(Q).view(bs, -1, self.n_heads, self.d_k).transpose(1,2) # q_s : [bs x n_heads x max_q_len x d_k]\n",
- " k_s = self.W_K(K).view(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1) # k_s : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)\n",
- " v_s = self.W_V(V).view(bs, -1, self.n_heads, self.d_v).transpose(1,2) # v_s : [bs x n_heads x q_len x d_v]\n",
+ " q_s = self.W_Q(Q).reshape(bs, -1, self.n_heads, self.d_k).transpose(1,2) # q_s : [bs x n_heads x max_q_len x d_k]\n",
+ " k_s = self.W_K(K).reshape(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1) # k_s : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)\n",
+ " v_s = self.W_V(V).reshape(bs, -1, self.n_heads, self.d_v).transpose(1,2) # v_s : [bs x n_heads x q_len x d_v]\n",
"\n",
" # Apply Scaled Dot-Product Attention (multiple heads)\n",
" if self.res_attention:\n",
@@ -3336,11 +3331,11 @@
" # output: [bs x n_heads x q_len x d_v], attn: [bs x n_heads x q_len x q_len], scores: [bs x n_heads x max_q_len x q_len]\n",
"\n",
" # back to the original inputs dimensions\n",
- " output = output.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * self.d_v) # output: [bs x q_len x n_heads * d_v]\n",
+ " output = output.transpose(1, 2).contiguous().reshape(bs, -1, self.n_heads * self.d_v) # output: [bs x q_len x n_heads * d_v]\n",
" output = self.to_out(output)\n",
"\n",
" if self.res_attention: return output, attn_weights, attn_scores\n",
- " else: return output, attn_weights "
+ " else: return output, attn_weights"
]
},
{
@@ -3367,7 +3362,7 @@
}
],
"source": [
- "q = torch.rand([16, 3, 50, 8]) \n",
+ "q = torch.rand([16, 3, 50, 8])\n",
"k = torch.rand([16, 3, 50, 8]).transpose(-1, -2)\n",
"v = torch.rand([16, 3, 50, 6])\n",
"attn_mask = torch.triu(torch.ones(50, 50)) # shape: q_len x q_len\n",
@@ -3450,7 +3445,7 @@
"loss = output[:2, :].sum()\n",
"test_eq(torch.isnan(loss).sum().item(), 0)\n",
"loss.backward()\n",
- "for n, p in mha.named_parameters(): \n",
+ "for n, p in mha.named_parameters():\n",
" if p.grad is not None:\n",
" test_eq(torch.isnan(p.grad).sum().item(), 0)"
]
@@ -3472,7 +3467,7 @@
"loss = output[:2, :].sum()\n",
"test_eq(torch.isnan(loss).sum().item(), 0)\n",
"loss.backward()\n",
- "for n, p in mha.named_parameters(): \n",
+ "for n, p in mha.named_parameters():\n",
" if p.grad is not None:\n",
" test_eq(torch.isnan(p.grad).sum().item(), 0)"
]
@@ -3597,18 +3592,18 @@
" cat_n_embeds = listify(n_cat_embeds)\n",
" if cat_padding_idxs is None: cat_padding_idxs = [None]\n",
" else: cat_padding_idxs = listify(cat_padding_idxs)\n",
- " if len(cat_padding_idxs) == 1 and len(cat_padding_idxs) < len(cat_n_embeds): \n",
+ " if len(cat_padding_idxs) == 1 and len(cat_padding_idxs) < len(cat_n_embeds):\n",
" cat_padding_idxs = cat_padding_idxs * len(cat_n_embeds)\n",
" assert len(cat_n_embeds) == len(cat_padding_idxs)\n",
- " if cat_embed_dims is None: \n",
+ " if cat_embed_dims is None:\n",
" cat_embed_dims = [emb_sz_rule(s) for s in cat_n_embeds]\n",
" else:\n",
" cat_embed_dims = listify(cat_embed_dims)\n",
" if len(cat_embed_dims) == 1: cat_embed_dims = cat_embed_dims * len(cat_n_embeds)\n",
" assert len(cat_embed_dims) == len(cat_n_embeds)\n",
- " if cat_pos: \n",
- " cat_pos = torch.as_tensor(listify(cat_pos)) \n",
- " else: \n",
+ " if cat_pos:\n",
+ " cat_pos = torch.as_tensor(listify(cat_pos))\n",
+ " else:\n",
" cat_pos = torch.arange(len(cat_n_embeds))\n",
" self.register_buffer(\"cat_pos\", cat_pos)\n",
" cont_pos = torch.tensor([p for p in torch.arange(c_in) if p not in self.cat_pos])\n",
@@ -3684,9 +3679,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "/Users/nacho/notebooks/tsai/nbs/029_models.layers.ipynb couldn't be saved automatically. You should save it manually 👋\n",
+ "/Users/nacho/notebooks/tsai/nbs/029_models.layers.ipynb saved at 2025-01-20 10:23:58\n",
"Correct notebook to script conversion! 😃\n",
- "Thursday 08/06/23 19:24:22 CEST\n"
+ "Monday 20/01/25 10:24:01 CET\n"
]
},
{
diff --git a/nbs/069_models.TSSequencerPlus.ipynb b/nbs/069_models.TSSequencerPlus.ipynb
index 56f6b5bc6..5f71ce661 100644
--- a/nbs/069_models.TSSequencerPlus.ipynb
+++ b/nbs/069_models.TSSequencerPlus.ipynb
@@ -44,11 +44,11 @@
"source": [
"#|export\n",
"class _TSSequencerEncoderLayer(nn.Module):\n",
- " def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0., \n",
+ " def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
" mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
" super().__init__()\n",
" self.bilstm = nn.LSTM(q_len, q_len, num_layers=1, bidirectional=True, bias=lstm_bias)\n",
- " self.dropout = nn.Dropout(lstm_dropout)\n",
+ " self.dropout = nn.Dropout(lstm_dropout) if lstm_dropout else nn.Identity()\n",
" self.fc = nn.Linear(2 * q_len, q_len)\n",
" self.lstm_norm = nn.LayerNorm(d_model)\n",
" self.pwff = PositionwiseFeedForward(d_model, dropout=dropout, act=act, mlp_ratio=mlp_ratio)\n",
@@ -75,7 +75,7 @@
"source": [
"#|export\n",
"class _TSSequencerEncoder(nn.Module):\n",
- " def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0., \n",
+ " def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
" mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
" super().__init__()\n",
" dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]\n",
@@ -101,22 +101,22 @@
"source": [
"#|export\n",
"class _TSSequencerBackbone(Module):\n",
- " def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu', \n",
- " lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, \n",
- " pre_norm:bool=False, use_token:bool=True, use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, \n",
- " cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None, \n",
+ " def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu',\n",
+ " lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1,\n",
+ " pre_norm:bool=False, use_token:bool=True, use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None,\n",
+ " cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None,\n",
" token_size:int=None, tokenizer:Optional[Callable]=None):\n",
"\n",
" # Categorical embeddings\n",
" if n_cat_embeds is not None:\n",
" n_cat_embeds = listify(n_cat_embeds)\n",
- " if cat_embed_dims is None: \n",
+ " if cat_embed_dims is None:\n",
" cat_embed_dims = [emb_sz_rule(s) for s in n_cat_embeds]\n",
" self.to_cat_embed = MultiEmbedding(c_in, n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos)\n",
" c_in, seq_len = output_size_calculator(self.to_cat_embed, c_in, seq_len)\n",
" else:\n",
" self.to_cat_embed = nn.Identity()\n",
- " \n",
+ "\n",
" # Sequence embedding\n",
" if token_size is not None:\n",
" self.tokenizer = SeqTokenizer(c_in, d_model, token_size)\n",
@@ -125,7 +125,7 @@
" if isinstance(tokenizer, nn.Module): self.tokenizer = tokenizer\n",
" else: self.tokenizer = tokenizer(c_in, d_model)\n",
" c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
- " else: \n",
+ " else:\n",
" self.tokenizer = nn.Identity()\n",
"\n",
" # Feature extractor\n",
@@ -135,13 +135,13 @@
" c_in, seq_len = output_size_calculator(self.feature_extractor, c_in, seq_len)\n",
" else:\n",
" self.feature_extractor = nn.Identity()\n",
- " \n",
+ "\n",
" # Linear projection\n",
" if token_size is None and tokenizer is None and feature_extractor is None:\n",
" self.linear_proj = nn.Conv1d(c_in, d_model, 1)\n",
" else:\n",
" self.linear_proj = nn.Identity()\n",
- " \n",
+ "\n",
" self.transpose = Transpose(1,2)\n",
"\n",
" # Position embedding & token\n",
@@ -150,10 +150,10 @@
" self.use_pe = use_pe\n",
" self.cls_token = nn.Parameter(torch.zeros(1, 1, d_model))\n",
" self.use_token = use_token\n",
- " self.emb_dropout = nn.Dropout(dropout)\n",
+ " self.emb_dropout = nn.Dropout(dropout) if dropout else nn.Identity()\n",
"\n",
" # Encoder\n",
- " self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias, \n",
+ " self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias,\n",
" lstm_dropout=lstm_dropout, dropout=dropout,\n",
" mlp_ratio=mlp_ratio, drop_path_rate=drop_path_rate, act=act, pre_norm=pre_norm)\n",
"\n",
@@ -161,19 +161,19 @@
"\n",
" # Categorical embeddings\n",
" x = self.to_cat_embed(x)\n",
- " \n",
+ "\n",
" # Sequence embedding\n",
" x = self.tokenizer(x)\n",
"\n",
" # Feature extractor\n",
" x = self.feature_extractor(x)\n",
- " \n",
+ "\n",
" # Linear projection\n",
" x = self.linear_proj(x)\n",
- " \n",
+ "\n",
" # Position embedding & token\n",
" x = self.transpose(x)\n",
- " if self.use_pe: \n",
+ " if self.use_pe:\n",
" x = x + self.pos_embed\n",
" if self.use_token: # token is concatenated after position embedding so that embedding can be learned using self.supervised learning\n",
" x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)\n",
@@ -181,7 +181,7 @@
"\n",
" # Encoder\n",
" x = self.encoder(x)\n",
- " \n",
+ "\n",
" # Output\n",
" x = x.transpose(1,2)\n",
" return x"
@@ -208,7 +208,7 @@
" depth: number of blocks in the encoder.\n",
" act: the activation function of positionwise feedforward layer.\n",
" lstm_dropout: dropout rate applied to the lstm sublayer.\n",
- " dropout: dropout applied to to the embedded sequence steps after position embeddings have been added and \n",
+ " dropout: dropout applied to to the embedded sequence steps after position embeddings have been added and\n",
" to the mlp sublayer in the encoder.\n",
" drop_path_rate: stochastic depth rate.\n",
" mlp_ratio: ratio of mlp hidden dim to embedding dim.\n",
@@ -224,38 +224,38 @@
" cat_pos: list with the position of the categorical variables in the input.\n",
" token_size: Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)\n",
" tokenizer: nn.Module or callable that will be used to reduce the sequence length\n",
- " feature_extractor: nn.Module or callable that will be used to preprocess the time series before \n",
+ " feature_extractor: nn.Module or callable that will be used to preprocess the time series before\n",
" the embedding step. It is useful to extract features or resample the time series.\n",
- " flatten: flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False. \n",
+ " flatten: flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.\n",
" If use_token is False and flatten is False, the model will apply a pooling layer.\n",
- " concat_pool: if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling. \n",
+ " concat_pool: if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.\n",
" fc_dropout: dropout applied to the final fully connected layer.\n",
" use_bn: flag that indicates if batchnorm will be applied to the head.\n",
" bias_init: values used to initialized the output layer.\n",
- " y_range: range of possible y values (used in regression tasks). \n",
+ " y_range: range of possible y values (used in regression tasks).\n",
" custom_head: custom head that will be applied to the network. It must contain all kwargs (pass a partial function)\n",
" verbose: flag to control verbosity of the model.\n",
"\n",
" Input:\n",
" x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)\n",
" \"\"\"\n",
- " \n",
+ "\n",
" def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',\n",
- " lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True, \n",
- " pre_norm:bool=False, use_token:bool=False, use_pe:bool=True, \n",
+ " lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,\n",
+ " pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,\n",
" cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,\n",
- " token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None, \n",
- " flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False, \n",
+ " token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,\n",
+ " flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,\n",
" bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,\n",
" **kwargs):\n",
"\n",
- " if use_token and c_out == 1: \n",
+ " if use_token and c_out == 1:\n",
" use_token = False\n",
" pv(\"use_token set to False as c_out == 1\", verbose)\n",
" backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,\n",
- " lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate, \n",
- " pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token, \n",
- " n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos, \n",
+ " lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,\n",
+ " pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,\n",
+ " n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,\n",
" feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)\n",
"\n",
" self.head_nf = d_model\n",
@@ -269,7 +269,7 @@
" else:\n",
" nf = d_model\n",
" layers = []\n",
- " if use_token: \n",
+ " if use_token:\n",
" layers += [TokenLayer()]\n",
" elif flatten:\n",
" layers += [Reshape(-1)]\n",
@@ -279,10 +279,10 @@
" layers = [GACP1d(1) if concat_pool else GAP1d(1)]\n",
" if use_bn: layers += [nn.BatchNorm1d(nf)]\n",
" if fc_dropout: layers += [nn.Dropout(fc_dropout)]\n",
- " \n",
+ "\n",
" # Last layer\n",
" linear = nn.Linear(nf, c_out)\n",
- " if bias_init is not None: \n",
+ " if bias_init is not None:\n",
" if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)\n",
" else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))\n",
" layers += [linear]\n",
@@ -290,8 +290,8 @@
" if y_range: layers += [SigmoidRange(*y_range)]\n",
" head = nn.Sequential(*layers)\n",
" super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))\n",
- " \n",
- " \n",
+ "\n",
+ "\n",
"TSSequencer = TSSequencerPlus"
]
},
@@ -403,7 +403,7 @@
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAABZcAAABoCAYAAACNDM73AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAa00lEQVR4nO3deXRU5eHG8edmshKyEAwhQbI0BkR2QkzZFA5LgpUKLohiG7AFLUuKKfgjHs2CCArKSdkEpCU9VSpqRamySiWAoqxBEQSExMSChioQA0KWmd8flpGRQHJJZoaE7+ecOWfmnXvv+8zoFfqct+8YNpvNJgAAAAAAAAAATPBwdwAAAAAAAAAAQMNDuQwAAAAAAAAAMI1yGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAA4CS5ubkyDEOFhYX2sb59+6pv3771PldWVpYMw3AYi46O1qhRo+p9rp8rLCyUYRjKzc21j40aNUpNmzZ1+twXGIahrKwsl80HAAAAgHIZAADA7tNPP9W9996rqKgo+fr6qlWrVho4cKDmzZvntDmPHTumrKws5efnO20OM1avXn3NlrTXcjYAAADgeuTp7gAAAADXgg8//FD9+vVTZGSkxowZo5YtW6q4uFgfffSR/vznP2vixIn1Ms/69esdXh87dkzZ2dmKjo5Wly5d6mWOCw4ePCgPD3NrCVavXq0FCxaYKnGjoqL0ww8/yMvLy2RCc66U7YcffpCnJ3+1BQAAAFyJv4EDAABIeuaZZxQUFKQdO3YoODjY4b2SkpJ6m8fb27verlUTHx8fp16/srJSVqtV3t7e8vX1depcNXH3/AAAAMD1iG0xAAAAJB05ckTt27e/pFiWpBYtWji8NgxDEyZM0CuvvKK2bdvK19dX8fHx2rx5c43zXLzn8qZNm5SQkCBJGj16tAzDuGTv4ups3bpVCQkJ8vX1VWxsrBYvXlztcT/fc7miokLZ2dmKi4uTr6+vmjdvrt69e2vDhg2SftwnecGCBfbPeOEh/bSv8vPPP6+cnBzFxsbKx8dH+/fvr3bP5QuOHj2qpKQk+fv7KyIiQtOmTZPNZrO/v2nTJhmGoU2bNjmc9/NrXinbhbGfr2jes2ePBg8erMDAQDVt2lT9+/fXRx995HDMhX2xP/jgA6WlpSk0NFT+/v4aNmyYTpw4Uf0/AAAAAACSWLkMAAAg6cetHbZt26Z9+/apQ4cONR6fl5enFStWKDU1VT4+Plq4cKGSk5O1ffv2Wp0vSe3atdO0adOUkZGhsWPHqk+fPpKknj17XvacTz/9VIMGDVJoaKiysrJUWVmpzMxMhYWF1ThfVlaWZs6cqd///ve69dZbVVpaqp07d2r37t0aOHCgHnnkER07dkwbNmzQ3//+92qvsWzZMp07d05jx46Vj4+PQkJCZLVaqz22qqpKycnJ+uUvf6lZs2Zp7dq1yszMVGVlpaZNm1aLb+gntcl2sc8++0x9+vRRYGCgHn/8cXl5eWnx4sXq27ev8vLylJiY6HD8xIkT1axZM2VmZqqwsFA5OTmaMGGCVqxYYSonAAAAcD2hXAYAAJA0efJkDR48WF26dNGtt96qPn36qH///urXr1+1ewnv27dPO3fuVHx8vCRpxIgRatu2rTIyMvTmm2/Was6wsDANHjxYGRkZ6tGjhx566KEaz8nIyJDNZtOWLVsUGRkpSbrnnnvUsWPHGs999913dccdd2jJkiXVvt+jRw+1adNGGzZsuGyWr776Sl988YVCQ0PtY4WFhdUee+7cOSUnJ2vu3LmSpHHjxmnIkCF67rnnlJqaqhtuuKHGzGayXezJJ59URUWFtm7dql/84heSpN/+9rdq27atHn/8ceXl5Tkc37x5c61fv96+GtpqtWru3Lk6ffq0goKCap0TAAAAuJ6wLQYAAICkgQMHatu2bfr1r3+tvXv3atasWUpKSlKrVq20atWqS47v0aOHvViWpMjISN11111at26dqqqqnJKxqqpK69at09ChQ+3FsvTjCuikpKQazw8ODtZnn32mw4cPX3WGe+65x6FYrsmECRPszy9sJ1JeXq733nvvqjPUpKqqSuvXr9fQoUPtxbIkhYeH68EHH9TWrVtVWlrqcM7YsWMdttno06ePqqqq9OWXXzotJwAAANDQUS4DAAD8T0JCgt58802dPHlS27dvV3p6ur7//nvde++92r9/v8OxcXFxl5zfpk0bnT171ml79Z44cUI//PBDtXO3bdu2xvOnTZumU6dOqU2bNurYsaOmTJmiTz75xFSGmJiYWh/r4eHhUO5KP35H0uVXO9eHEydO6OzZs9V+J+3atZPValVxcbHD+MVlvSQ1a9ZMknTy5Emn5QQAAAAaOsplAACAn/H29lZCQoJmzJihF198URUVFXr99dfdHavObrvtNh05ckR//etf1aFDBy1dulTdunXT0qVLa30NPz+/es108Wrhizlr9fflWCyWascv/vFBAAAAAI4olwEAAK6ge/fukqTjx487jFe3tcShQ4fUpEkTU9tGXK5crU5oaKj8/PyqnfvgwYO1ukZISIhGjx6tf/zjHyouLlanTp2UlZV1VXlqYrVadfToUYexQ4cOSZKio6Ml/bRC+NSpUw7HVbcdRW2zhYaGqkmTJtV+J59//rk8PDzUunXrWl0LAAAAwOVRLgMAAEh6//33q12lunr1akmXbjuxbds27d692/66uLhYb7/9tgYNGnTZVbDV8ff3l3RpuVodi8WipKQkvfXWWyoqKrKPHzhwQOvWravx/G+//dbhddOmTXXTTTfp/PnzV5WnNubPn29/brPZNH/+fHl5eal///6SpKioKFksFm3evNnhvIULF15yrdpms1gsGjRokN5++22H7Te++eYbLV++XL1791ZgYOBVfiIAAAAAF3i6OwAAAMC1YOLEiTp79qyGDRumm2++WeXl5frwww+1YsUKRUdHa/To0Q7Hd+jQQUlJSUpNTZWPj4+9DM3OzjY1b2xsrIKDg7Vo0SIFBATI399fiYmJl93bODs7W2vXrlWfPn00btw4VVZWat68eWrfvn2N+yffcsst6tu3r+Lj4xUSEqKdO3fqjTfecPjRvQs/UpiamqqkpCRZLBaNGDHC1Ge6wNfXV2vXrlVKSooSExO1Zs0avfvuu3riiSfsq7uDgoJ03333ad68eTIMQ7GxsXrnnXdUUlJyyfXMZJs+fbo2bNig3r17a9y4cfL09NTixYt1/vx5zZo166o+DwAAAABHlMsAAACSnn/+eb3++utavXq1lixZovLyckVGRmrcuHF68sknFRwc7HD87bffrh49eig7O1tFRUW65ZZblJubq06dOpma18vLS3/729+Unp6uRx99VJWVlVq2bNlly+VOnTpp3bp1SktLU0ZGhm688UZlZ2fr+PHjNZbLqampWrVqldavX6/z588rKipK06dP15QpU+zH3H333Zo4caJeffVVvfzyy7LZbFddLlssFq1du1Z/+MMfNGXKFAUEBCgzM1MZGRkOx82bN08VFRVatGiRfHx8NHz4cM2ePVsdOnRwOM5Mtvbt22vLli1KT0/XzJkzZbValZiYqJdfflmJiYlX9XkAAAAAODJs/EoJAACAKYZhaPz48Q5bPgAAAADA9YY9lwEAAAAAAAAAplEuAwAAAAAAAABMo1wGAAAAAAAAAJjGD/oBAACYxE9WAAAAAAArlwEAAAAAAAAAV4FyGQAAAAAAAABgmsu3xbBarTp27JgCAgJkGIarpwcAAAAAAAAaNJvNpu+//14RERHy8GDtKNzH5eXysWPH1Lp1a1dPCwAAAAAAADQqxcXFuvHGG90dA9cxl5fLAQEB/3tWLCnQ1dMDcLLOebe5OwIAJ9l7+2Z3RwAAAAAgSSqV1Pqing1wD5eXyz9thREoymWg8bE0tbg7AgCn4c9tAAAA4FrClrNwNzZlAQAAAAAAAACYRrkMAAAAAAAAADCNchkAAAAAAAAAYJrL91wGAAAAAAAAAGeoqqpSRUWFu2M0WBaLRZ6enrXez5tyGQAAAAAAAECDV1ZWpq+++ko2m83dURq0Jk2aKDw8XN7e3jUeS7kMAAAAAAAAoEGrqqrSV199pSZNmig0NLTWK2/xE5vNpvLycp04cUIFBQWKi4uTh8eVd1WmXAYAAAAAAADQoFVUVMhmsyk0NFR+fn7ujtNg+fn5ycvLS19++aXKy8vl6+t7xeP5QT8AAAAAAAAAjQIrluuuptXKDsc6MQcAAAAAAAAAoJGiXAYAAAAAAAAAmEa5DAAAAAAAAACNRHR0tHJyclwyF+UyAAAAAAAAgEbJMFz7MJfNuOIjKyvrqj7zjh07NHbs2Ks61yzT5fLmzZs1ZMgQRUREyDAMvfXWW06IBQAAAAAAAACN1/Hjx+2PnJwcBQYGOoxNnjzZfqzNZlNlZWWtrhsaGqomTZo4K7YD0+XymTNn1LlzZy1YsMAZeQAAAAAAAACg0WvZsqX9ERQUJMMw7K8///xzBQQEaM2aNYqPj5ePj4+2bt2qI0eO6K677lJYWJiaNm2qhIQEvffeew7X/fm2GIZhaOnSpRo2bJiaNGmiuLg4rVq1ql4+g+lyefDgwZo+fbqGDRtWLwEAAAAAAAAAAJeaOnWqnn32WR04cECdOnVSWVmZ7rjjDm3cuFF79uxRcnKyhgwZoqKioiteJzs7W8OHD9cnn3yiO+64QyNHjtR3331X53xO33P5/PnzKi0tdXgAAAAAAAAAAK5s2rRpGjhwoGJjYxUSEqLOnTvrkUceUYcOHRQXF6enn35asbGxNa5EHjVqlB544AHddNNNmjFjhsrKyrR9+/Y653N6uTxz5kwFBQXZH61bt3b2lAAAAAAAAADQ4HXv3t3hdVlZmSZPnqx27dopODhYTZs21YEDB2pcudypUyf7c39/fwUGBqqkpKTO+ZxeLqenp+v06dP2R3FxsbOnBAAAAAAAAIAGz9/f3+H15MmTtXLlSs2YMUNbtmxRfn6+OnbsqPLy8itex8vLy+G1YRiyWq11zudZ5yvUwMfHRz4+Ps6eBgAAAAAAAAAatQ8++ECjRo2y/x5eWVmZCgsL3ZbH6SuXAQAAAAAAAAB1FxcXpzfffFP5+fnau3evHnzwwXpZgXy1TK9cLisr0xdffGF/XVBQoPz8fIWEhCgyMrJewwEAAAAAAADA1bLZ3J2gfs2ZM0cPP/ywevbsqRtuuEH/93//p9LSUrflMWw2c1/xpk2b1K9fv0vGU1JSlJubW+P5paWlCgoKknRaUqCZqQE0AN12xbs7AgAn2R2/y90RAAAAAEiSSiUF6fTp0woMpF+TpHPnzqmgoEAxMTHy9fV1d5wGzcx3aXrlct++fWWyjwYAAAAAAAAANDLsuQwAAAAAAAAAMI1yGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAAAAAAAAAwzdPdAQAAAAAAAADAGeJ3x7t0vl3ddtX6WMMwrvh+ZmamsrKyriqHYRhauXKlhg4delXn1xblMgAAAAAAAAC42PHjx+3PV6xYoYyMDB08eNA+1rRpU3fEMsXl5bLNZvvfs1JXTw3ABarKqtwdAYDT8Gc3AAAAcG348e/mP/VsaIhatmxpfx4UFCTDMBzGli5dqhdeeEEFBQWKjo5Wamqqxo0bJ0kqLy9XWlqa/vnPf+rkyZMKCwvTo48+qvT0dEVHR0uShg0bJkmKiopSYWGhUz6Dy8vlb7/99n/PWrt6agAusPd2dycA4DxB7g4AAAAA4CLffvutgoL4e3pj9MorrygjI0Pz589X165dtWfPHo0ZM0b+/v5KSUnR3LlztWrVKr322muKjIxUcXGxiouLJUk7duxQixYttGzZMiUnJ8tisTgtp8vL5ZCQEElSUVER//IDjUxpaalat26t4uJiBQYGujsOgHrE/Q00XtzfQOPF/Q00XqdPn1ZkZKS9Z0Pjk5mZqRdeeEF33323JCkmJkb79+/X4sWLlZKSoqKiIsXFxal3794yDENRUVH2c0NDQyVJwcHBDiuhncHl5bKHh4ekH5d684cb0DgFBgZyfwONFPc30HhxfwONF/c30Hhd6NnQuJw5c0ZHjhzR7373O40ZM8Y+XllZaV+sO2rUKA0cOFBt27ZVcnKy7rzzTg0aNMjlWflBPwAAAAAAAAC4RpSVlUmSXnrpJSUmJjq8d2GLi27duqmgoEBr1qzRe++9p+HDh2vAgAF64403XJqVchkAAAAAAAAArhFhYWGKiIjQ0aNHNXLkyMseFxgYqPvvv1/333+/7r33XiUnJ+u7775TSEiIvLy8VFVV5fSsLi+XfXx8lJmZKR8fH1dPDcDJuL+Bxov7G2i8uL+Bxov7G2i8uL8bv+zsbKWmpiooKEjJyck6f/68du7cqZMnTyotLU1z5sxReHi4unbtKg8PD73++utq2bKlgoODJUnR0dHauHGjevXqJR8fHzVr1swpOQ2bzWZzypUBAAAAAAAAwAXOnTungoICxcTEyNfX191xTMvNzdWkSZN06tQp+9jy5cs1e/Zs7d+/X/7+/urYsaMmTZqkYcOG6aWXXtLChQt1+PBhWSwWJSQkaPbs2eratask6V//+pfS0tJUWFioVq1aqbCwsNZZzHyXlMsAAAAAAAAAGrSGXi5fS8x8l/ykJAAAAAAAAADANMplAAAAAAAAAIBplMsAAAAAAAAAANMolwEAAAAAAAAAprm0XF6wYIGio6Pl6+urxMREbd++3ZXTA3CCmTNnKiEhQQEBAWrRooWGDh2qgwcPujsWACd49tlnZRiGJk2a5O4oAOrBf/7zHz300ENq3ry5/Pz81LFjR+3cudPdsQDUUVVVlZ566inFxMTIz89PsbGxevrpp2Wz2dwdDYBJmzdv1pAhQxQRESHDMPTWW285vG+z2ZSRkaHw8HD5+flpwIABOnz4sHvCXkP4713dmfkOXVYur1ixQmlpacrMzNTu3bvVuXNnJSUlqaSkxFURADhBXl6exo8fr48++kgbNmxQRUWFBg0apDNnzrg7GoB6tGPHDi1evFidOnVydxQA9eDkyZPq1auXvLy8tGbNGu3fv18vvPCCmjVr5u5oAOroueee04svvqj58+frwIEDeu655zRr1izNmzfP3dEAmHTmzBl17txZCxYsqPb9WbNmae7cuVq0aJE+/vhj+fv7KykpSefOnXNx0muDxWKRJJWXl7s5ScN39uxZSZKXl1eNxxo2F9X5iYmJSkhI0Pz58yVJVqtVrVu31sSJEzV16lRXRADgAidOnFCLFi2Ul5en2267zd1xANSDsrIydevWTQsXLtT06dPVpUsX5eTkuDsWgDqYOnWqPvjgA23ZssXdUQDUszvvvFNhYWH6y1/+Yh+755575Ofnp5dfftmNyQDUhWEYWrlypYYOHSrpx5WlERER+tOf/qTJkydLkk6fPq2wsDDl5uZqxIgRbkzrHjabTUVFRaqoqFBERIQ8PNgN2CybzaazZ8+qpKREwcHBCg8Pr/EcTxfkUnl5uXbt2qX09HT7mIeHhwYMGKBt27a5IgIAFzl9+rQkKSQkxM1JANSX8ePH61e/+pUGDBig6dOnuzsOgHqwatUqJSUl6b777lNeXp5atWqlcePGacyYMe6OBqCOevbsqSVLlujQoUNq06aN9u7dq61bt2rOnDnujgagHhUUFOjrr7/WgAED7GNBQUFKTEzUtm3brsty2TAMhYeHq6CgQF9++aW74zRowcHBatmyZa2OdUm5/N///ldVVVUKCwtzGA8LC9Pnn3/uiggAXMBqtWrSpEnq1auXOnTo4O44AOrBq6++qt27d2vHjh3ujgKgHh09elQvvvii0tLS9MQTT2jHjh1KTU2Vt7e3UlJS3B0PQB1MnTpVpaWluvnmm2WxWFRVVaVnnnlGI0eOdHc0APXo66+/lqRqu7YL712PvL29FRcXx9YYdeDl5WXfYqQ2XFIuA7g+jB8/Xvv27dPWrVvdHQVAPSguLtYf//hHbdiwQb6+vu6OA6AeWa1Wde/eXTNmzJAkde3aVfv27dOiRYsol4EG7rXXXtMrr7yi5cuXq3379srPz9ekSZMUERHB/Q3guuDh4cH/fnEhl2w+csMNN8hiseibb75xGP/mm29qvcQawLVtwoQJeuedd/T+++/rxhtvdHccAPVg165dKikpUbdu3eTp6SlPT0/l5eVp7ty58vT0VFVVlbsjArhK4eHhuuWWWxzG2rVrp6KiIjclAlBfpkyZoqlTp2rEiBHq2LGjfvOb3+ixxx7TzJkz3R0NQD260KfRtcHdXFIue3t7Kz4+Xhs3brSPWa1Wbdy4UT169HBFBABOYrPZNGHCBK1cuVL//ve/FRMT4+5IAOpJ//799emnnyo/P9/+6N69u0aOHKn8/HxT/1cpANeWXr166eDBgw5jhw4dUlRUlJsSAagvZ8+eveRHrCwWi6xWq5sSAXCGmJgYtWzZ0qFrKy0t1ccff0zXBpdy2bYYaWlpSklJUffu3XXrrbcqJydHZ86c0ejRo10VAYATjB8/XsuXL9fbb7+tgIAA+95OQUFB8vPzc3M6AHUREBBwyf7p/v7+at68OfuqAw3cY489pp49e2rGjBkaPny4tm/friVLlmjJkiXujgagjoYMGaJnnnlGkZGRat++vfbs2aM5c+bo4Ycfdnc0ACaVlZXpiy++sL8uKChQfn6+QkJCFBkZqUmTJmn69OmKi4tTTEyMnnrqKUVERGjo0KHuC43rjmGz2Wyummz+/PmaPXu2vv76a3Xp0kVz585VYmKiq6YH4ASGYVQ7vmzZMo0aNcq1YQA4Xd++fdWlSxfl5OS4OwqAOnrnnXeUnp6uw4cPKyYmRmlpaRozZoy7YwGoo++//15PPfWUVq5cqZKSEkVEROiBBx5QRkaGvL293R0PgAmbNm1Sv379LhlPSUlRbm6ubDabMjMztWTJEp06dUq9e/fWwoUL1aZNGzekxfXKpeUyAAAAAAAAAKBxcMmeywAAAAAAAACAxoVyGQAAAAAAAABgGuUyAAAAAAAAAMA0ymUAAAAAAAAAgGmUywAAAAAAAAAA0yiXAQAAAAAAAACmUS4DAAAAAAAAAEyjXAYAAAAAAAAAmEa5DAAAAAAAAAAwjXIZAAAAAAAAAGAa5TIAAAAAAAAAwLT/B0QiwpYKlPS/AAAAAElFTkSuQmCC",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAABZcAAABoCAYAAACNDM73AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGB1JREFUeJzt3QtUlHX+x/EvN0W5KV5QFMU1LPOaaKa5pqc0a7OkzLWs1Mouluwu2YVOgZZJZXnMC6XrJp3SspvGVrapm6J28W4XlbxLm6WVgliK4vzP9/dvaNAReICZhxner3PmDDPMPM93HnwEPnzn+wtwOBwOAQAAAAAAAADAgkArDwYAAAAAAAAAQBEuAwAAAAAAAAAsI1wGAAAAAAAAAFhGuAwAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgGWEywAAAAAAAAAAywiXAQAAAAAAAACWES4DAAB4SFZWlgQEBMjevXtL7uvXr5+5VLcJEyaYfbmKj4+XUaNGiafp69N96+t10v2Gh4eLt+j+9RgAAAAA8B7CZQAAgN999dVXMnToUGndurWEhoZKixYtZMCAATJjxgyP7fP77783oejmzZulJvjwww9rbEhbk2sDAAAAaqNguwsAAACoCT799FPp37+/tGrVSsaMGSPNmjWTvLw8+fzzz+WFF16QcePGVct+Pv7447PC5YkTJ5ou465du0p1ys3NlcDAQMsB7qxZsyyFuBrG//bbbxISElKJKqunNt1/cDA/2gIAAADexE/gAAAAIvLUU09JVFSUrFu3Tho0aFDqcwcPHqy2/dSpU0e8pW7duh7d/qlTp+T06dPmNWmnt53s3j8AAABQGzEWAwAAQER27dolHTp0OCtYVk2bNj1rvu/9998v8+fPl/PPP98Em4mJiZKTk1PuflxnLq9YsUJ69OhhPh49erTZ7pmzi91ZvXq1eZ7ut23btjJ79my3jztz5vLJkydNl3RCQoJ5bqNGjaRPnz6ydOlS83l9rHYGO1+j8+I6V/m5556TadOmmf1qeL1161a3M5eddu/eLVdeeaWEhYVJbGysPPHEE+JwOEo+r8dAn6vXrs7cZlm1Oe87s6N506ZNctVVV0lkZKSZ/3z55ZebTnR3c7HXrFkjKSkp0qRJE1NrUlKSHDp0qMyvAwAAAFDb0bkMAADw+2iHzz77TL7++mvp2LFjuY9fuXKlLFy4UJKTk03ImpmZKYMGDZK1a9dW6Pmqffv2JmxNS0uTu+66S/785z+b+3v37l3mXOiBAweaEFTDVO0eTk9Pl5iYmHL3p4/PyMiQO++8Uy6++GIpKCiQ9evXy8aNG81s6bvvvtuM6dCw+dVXX3W7jXnz5snx48dNvfq6o6OjTfeyO8XFxeaYXHLJJfLss8/KRx99ZGrVmvV1W1GR2lx988035nhqsPzQQw+ZkR0awmuwr1+7nj17lnq8jj1p2LChqU+DbQ3Q9Q8I+jUGAAAA4B7hMgAAgIiMHz/edLnq3GMNXjWY1E5XncPsbpawhtAazGrHsho+fLjpYtag+N13363QPjUQ1n3qc3r16iW33HJLuc/Rx2rn76pVq8x8aHXDDTdIp06dyn3uBx98IFdffbXMmTPH7ee1hnbt2pkA91y1fPfdd7Jz504TbjtpGOuOhtAaLk+fPt3cHjt2rAwePFieeeYZE8o3bty43Jqt1ObqscceM53a2uX9pz/9ydx32223ma+Rhs0aMLvSLm6dh+3shtbAXOvOz88341IAAAAAnI2xGAAAACKmc1c7l6+99lrZsmWL6bTVcQ4tWrSQ7Oxst2GnM1hWGvRed9118p///Md07HqCble3P2TIkJJg2dkBrbWWR0d+aEfvjh07Kl2DBtmuwXJ5tPv3zHEiRUVFsmzZMvEUPU4aFOtxcgbLqnnz5nLzzTebwFm7tl1pJ7brmA3944JuZ9++fR6rEwAAAPB1hMsAAAC/0znG2nV8+PBhM94iNTVVjh49KkOHDjWzhV3p3OIzaWftr7/+6rFZvbrd3377ze2+tSO3PDqK4siRI6ZO7XR+8MEH5csvv7RUQ5s2bSr82MDAwFLhrtJ9l9XtXF3HSb8O7o6JBvHalZyXl1fqftewXumIDKX/FgAAAAC4R7gMAABwhjp16pigefLkyfLiiy+a8QpvvfWW+Lq+ffuahQtffvllMxd67ty50q1bN3NdUfXq1avWmly7hV15qvv7XIKCgtze77r4IAAAAIDSCJcBAADK0L17d3N94MCBUve7Gy3x7bffSv369S2NjThXuOqOblfDXXf7zs3NrdA2dAG+0aNHy+uvv266dzt37mwW+qtMPeXRDuHdu3efdYxUfHx8qQ5h7ah25W4cRUVr0+OkXwd3x2T79u2mozouLs7CKwEAAADgDuEyAACAiHzyySduu1Q//PBDc33miAWdz7xx48aS2xrUvvfeezJw4MBzdsG6ExYW5jZcdUe3q7OVFy9eLPv37y+5f9u2bWYWc3l+/vnnUrfDw8PlvPPOkxMnTlSqnoqYOXNmycd6fPW2LpCoiyWq1q1bm9eVk5NT6nmZmZlnbauiten29OugXw/X8Rs//vijLFiwQPr06SORkZFVfm0AAABAbRdsdwEAAAA1wbhx48yc3qSkJLngggvMonOffvqpLFy40HTZarevKx0roUFvcnKy1K1btyQMnThxoqX9tm3b1iy099JLL0lERIQJUHv27HnO2ca6/Y8++sgsODd27Fg5deqUzJgxQzp06FDu/OQLL7xQ+vXrZxYi1A7m9evXy9tvv11q0T3nIoX6uvT1aVA7fPhwqYzQ0FBT68iRI81rWrJkiXzwwQfy6KOPlnR3R0VFyY033mheg3Ym6/F4//335eDBg2dtz0ptkyZNkqVLl5ogWY9TcHCwzJ492wTpulgjAAAAgKojXAYAABCR5557zsxV1k7lOXPmmHBZF3nTYPKxxx4zAbCryy67THr16mXCXu0i1uA2KyvLjJmwQrt4X3nlFbN44D333GPC4nnz5p0zXNbta5dySkqKpKWlScuWLU0NOrajvHBZQ9ns7Gz5+OOPTciqXcMawurCfk7XX3+9CdrfeOMNee2110y3cWXDZQ1/NVy+9957zT40PE9PTzd1u9JgWedaa8CuQf2wYcNkypQpJsB3ZaU2DdtXrVpljmtGRoYZ0aEBtz5PrwEAAABUXYCDVUoAAAAs0Q7b++67r9TIBwAAAACobZi5DAAAAAAAAACwjHAZAAAAAAAAAGAZ4TIAAAAAAAAAwDIW9AMAALCIJSsAAAAAgM5lAAAAAAAAAEAlEC4DAAAAAAAAAGr+WIzTp0/L999/LxERERIQEODt3QMAAAAAAAA+P6bt6NGjEhsbK4GB9I6iFoXLGizHxcV5e7cAAAAAAACAX8nLy5OWLVvaXQZqMa+Hy9qx/P/yRCTS27sHAAAAcIYuK/vaXQIAD9lyWY7dJQDwiAIRiXPJ2YBaEi7/MQpDg2XCZQAAAMBuQeFBdpcAwGP4vRvwZ4ychd0YygIAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAKDmz1wGAAAAAAAAAE8oLi6WkydP2l2GzwoKCpLg4OAKz/MmXAYAAAAAAADg8woLC+W7774Th8Nhdyk+rX79+tK8eXOpU6dOuY8lXAYAAAAAAADg8x3LGixrMNqkSZMKd97iDxrKFxUVyaFDh2TPnj2SkJAggYFlT1UmXAYAAAAAAADg03QUhoajGizXq1fP7nJ8lh67kJAQ2bdvnwmaQ0NDy3w8C/oBAAAAAAAA8At0LFdded3KpR5bDfsDAAAAAAAAANQyhMsAAAAAAAAAAMsIlwEAAAAAAADAT8THx8u0adO8si/CZQAAAAAAAAB+SUcwe/NidT50WZcJEyZIZaxbt07uuusuqZHhck5OjgwePFhiY2PNi1y8eLFnKgMAAAAAAAAAP3XgwIGSi3YaR0ZGlrpv/PjxJY91OBxy6tSpCm23SZMmUr9+famR4fKxY8ekS5cuMmvWLM9UBAAAAAAAAAB+rlmzZiWXqKgo08jrvL19+3aJiIiQJUuWSGJiotStW1dWr14tu3btkuuuu05iYmIkPDxcevToIcuWLStzLIZud+7cuZKUlGRC54SEBMnOzrYnXL7qqqtk0qRJphgAAAAAAAAAgGc88sgj8vTTT8u2bdukc+fOUlhYKFdffbUsX75cNm3aJIMGDTJTJvbv31/mdiZOnCjDhg2TL7/80jx/xIgR8ssvv9T8mcsnTpyQgoKCUhcAAAAAAAAAQNmeeOIJGTBggLRt21aio6PNRIm7775bOnbsaDqQn3zySfO58jqRR40aJTfddJOcd955MnnyZBNSr127Vmp8uJyRkWHaup2XuLg4T+8SAAAAAAAAAHxe9+7dS93WUFhnMbdv314aNGhgRmNoV3N5ncva9ewUFhZm5jsfPHiw5ofLqampkp+fX3LJy8vz9C4BAAAAAAAAwOeFhYWVuq3B8qJFi0z38apVq2Tz5s3SqVMnKSoqKnM7ISEhpW7rHObTp09Xub5g8TAdNq0XAAAAAAAAAEDlrVmzxoy4cK6Hp53Me/fuFbt4vHMZAAAAAAAAAFB1Omf53XffNR3LW7ZskZtvvrlaOpC91rmsafjOnTtLbu/Zs8e8GB0o3apVq+quDwAAAAAAAAAqxeEQvzJ16lS5/fbbpXfv3tK4cWN5+OGHpaCgwLZ6AhwOa4d4xYoV0r9//7PuHzlypGRlZZX7fH2xurCfSL6IRFqrFgAAAEC167Yh0e4SAHjIxsQNdpcAwCM0TIwy65vpwmwQOX78uGmCbdOmjYSGhtpdTq05lpY7l/v16ycW82gAAAAAAAAAgJ9h5jIAAAAAAAAAwDLCZQAAAAAAAACAZYTLAAAAAAAAAADLCJcBAAAAAAAAAJYRLgMAAAAAAAAALCNcBgAAAAAAAABYRrgMAAAAAAAAALCMcBkAAAAAAAAAYBnhMgAAAAAAAADAsmDrTwEAAAAAAACAmi9xY6JX97eh24YKPzYgIKDMz6enp8uECRMqVYdue9GiRTJkyBDxJMJlAAAAAAAAAPCyAwcOlHy8cOFCSUtLk9zc3JL7wsPDpabzerjscDh+/6jA27sGAAAA4EZxYbHdJQDwGH73Bvz53P4jZ4MvatasWcnHUVFRptvY9b65c+fK888/L3v27JH4+HhJTk6WsWPHms8VFRVJSkqKvPPOO3L48GGJiYmRe+65R1JTU81jVVJSkrlu3bq17N271z/C5Z9//vn3j+K8vWsAAAAAbmy5zO4KAHhOlN0FAPBwzqahJPzP/PnzTSfzzJkz5aKLLpJNmzbJmDFjJCwsTEaOHCnTp0+X7OxsefPNN6VVq1aSl5dnLmrdunXStGlTmTdvngwaNEiCgoI8VqfXw+Xo6GhzvX//fv7xA36moKBA4uLizH9mkZGRdpcDoBpxfgP+i/Mb8F+c34D/ys/PN4GiM2eD/0lPTzddy9dff7253aZNG9m6davMnj3bhMuarSYkJEifPn1Mx7N2Jzs1adLEXDdo0KBUJ7RfhMuBgYHmWoNlvrkB/knPbc5vwD9xfgP+i/Mb8F+c34D/cuZs8C/Hjh2TXbt2yR133GG6lZ1OnTpV0qw7atQoGTBggJx//vmmO/maa66RgQMHer1WFvQDAAAAAAAAgBqisLDQXP/zn/+Unj17lvqcc8RFt27dzCzmJUuWyLJly2TYsGFyxRVXyNtvv+3VWgmXAQAAAAAAAKCGiImJkdjYWNm9e7eMGDHinI/Td6X89a9/NZehQ4eaDuZffvnFjEsJCQmR4uJi/wuX69ata2aG6DUA/8L5Dfgvzm/Af3F+A/6L8xvwX5zf/m/ixImSnJxsxmBoaHzixAlZv369HD58WFJSUmTq1KnSvHlzs9ifjkd56623zHxlnbOs4uPjZfny5XLppZeafycNGzb0SJ0BDofD4ZEtAwAAAAAAAIAXHD9+3IyJ0IXvQkNDxddkZWXJ3//+dzly5EjJfQsWLJApU6aYhfzCwsKkU6dO5jFJSUlmZEZmZqbs2LHDjMro0aOHeayGzerf//63CaH37t0rLVq0MNeeOJaEywAAAAAAAAB8mq+Hy756LFlSEgAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgGWEywAAAAAAAACAmh0uz5o1S+Lj480g6J49e8ratWu9uXsAHpCRkWFWJI2IiJCmTZvKkCFDJDc31+6yAHjA008/LQEBAWZ1YgC+73//+5/ccsst0qhRI6lXr55ZfXz9+vV2lwWgioqLi+Xxxx83izDpud22bVt58sknxeFw2F0aAItycnJk8ODBEhsba34OX7x4canP63mdlpYmzZs3N+f7FVdcITt27JDajv/vvHsMvRYuL1y4UFJSUiQ9PV02btwoXbp0kSuvvFIOHjzorRIAeMDKlSvlvvvuk88//1yWLl0qJ0+elIEDB8qxY8fsLg1ANVq3bp3Mnj1bOnfubHcpAKrB4cOH5dJLL5WQkBBZsmSJbN26VZ5//nlp2LCh3aUBqKJnnnlGXnzxRZk5c6Zs27bN3H722WdlxowZdpcGwCL9vVrzM23WdEfP7enTp8tLL70kX3zxhYSFhZms7fjx41IbBQUFmeuioiK7S/F5v/76q7nWnxXLE+DwUpyvncra3ajf4NTp06clLi5Oxo0bJ4888og3SgDgBYcOHTIdzBo69+3b1+5yAFSDwsJC6datm2RmZsqkSZOka9euMm3aNLvLAlAF+vP3mjVrZNWqVXaXAqCaXXPNNRITEyP/+te/Su674YYbTFfja6+9ZmttACpPO5cXLVpk3i2sNM7TjuYHHnhAxo8fb+7Lz883539WVpYMHz5cahs9Jvv37zdNb3psAgOZBlyZY6jBsjYDN2jQwHTFlydYvED/YrBhwwZJTU0tuU+/wNqu/9lnn3mjBABeot/MVHR0tN2lAKgm+u6Ev/zlL+b7tobLAHxfdna26Wy68cYbzR+EW7RoIWPHjpUxY8bYXRqAKurdu7fMmTNHvv32W2nXrp1s2bJFVq9eLVOnTrW7NADVaM+ePfLDDz+Yn9GdoqKiTHOnZm21MVzWAF7DUD02+/bts7scn6bBcrNmzSr0WK+Eyz/99JOZ+6R/PXGlt7dv3+6NEgB4gb4jQWex6ttsO3bsaHc5AKrBG2+8YcZZ6VgMAP5j9+7d5m3zOrbu0UcfNed4cnKy1KlTR0aOHGl3eQCq+M6EgoICueCCC8xbxPV38aeeekpGjBhhd2kAqpEGy8pd1ub8XG2kP8skJCQwGqMKdBSGc8RIjQmXAdSe7savv/7adEYA8H15eXnyt7/9zcxT18V4AfjXH4S7d+8ukydPNrcvuugi8z1cZzYSLgO+7c0335T58+fLggULpEOHDrJ582bTAKJvEef8BlAb6LQEfn/xHq8MH2ncuLFJvH/88cdS9+vtirZYA6jZ7r//fnn//fflk08+kZYtW9pdDoBqoCOtdNaWzlsODg42F337vC4aoh9rJxQA36RvGb3wwgtL3de+fXszpxCAb3vwwQdN97K+Jb5Tp05y6623yj/+8Q/JyMiwuzQA1ciZp5G1oVaEy9qSnpiYKMuXLy/VLaG3e/Xq5Y0SAHhw2LsGy7qwwH//+19p06aN3SUBqCaXX365fPXVV6bjyXnRTkd9W61+bOWtUgBqFh1hlZubW+o+nc/aunVr22oCUD10IaYzF7HS79n6OzgA/6G/e2uI7Jq16UicL774gqwNXuW1sRg6z03fgqO/lF588cVmlfljx47J6NGjvVUCAA+NwtC33L333nsSERFRMttJFxLQFakB+C49p8+cnx4WFiaNGjVirjrg47SLURf90rEYw4YNk7Vr15oFwPQCwLcNHjzYzFhu1aqVGYuxadMms5jf7bffbndpACwqLCyUnTt3ltzWheq0ySM6Otqc4zryRhfc1hnDGjY//vjjZgTOkCFDbK0btUuAQ9sOvWTmzJkyZcoUEz517drVvK1WV7EE4Nursbozb948GTVqlNfrAeBZ/fr1M9/D9Y/EAHybjrNKTU2VHTt2mF9ItRlkzJgxdpcFoIqOHj1qAiZ9Z6GOt9Kg6aabbpK0tDTzrmIAvmPFihXSv3//s+7X5s2srCzzTuL09HTzx+EjR45Inz59JDMzU9q1a2dLvaidvBouAwAAAAAAAAD8g1dmLgMAAAAAAAAA/AvhMgAAAAAAAADAMsJlAAAAAAAAAIBlhMsAAAAAAAAAAMsIlwEAAAAAAAAAlhEuAwAAAAAAAAAsI1wGAAAAAAAAAFhGuAwAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgFj1fw7YwpaVgPHmAAAAAElFTkSuQmCC",
"text/plain": [
""
]
@@ -414,7 +414,7 @@
{
"data": {
"text/plain": [
- "TSTensor(samples:8, vars:3, len:5000, device=cpu, dtype=torch.float32)"
+ "TSTensor(samples:8, vars:3, len:5000, device=mps:0, dtype=torch.float32)"
]
},
"execution_count": null,
@@ -423,7 +423,7 @@
}
],
"source": [
- "X = np.zeros((10, 3, 5000)) \n",
+ "X = np.zeros((10, 3, 5000))\n",
"y = np.random.randint(0,2,X.shape[0])\n",
"splits = get_splits(y)\n",
"dls = get_ts_dls(X, y, splits=splits)\n",
@@ -562,15 +562,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "[W NNPACK.cpp:53] Could not initialize NNPACK! Reason: Unsupported hardware.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"a = alphabet[np.random.randint(0,3,40)]\n",
"b = ALPHABET[np.random.randint(6,10,40)]\n",
@@ -663,9 +655,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "/Users/nacho/notebooks/tsai/nbs/069_models.TSSequencerPlus.ipynb saved at 2023-03-26 16:01:39\n",
+ "/Users/nacho/notebooks/tsai/nbs/069_models.TSSequencerPlus.ipynb saved at 2025-01-20 10:26:55\n",
"Correct notebook to script conversion! 😃\n",
- "Sunday 26/03/23 16:01:41 CEST\n"
+ "Monday 20/01/25 10:26:58 CET\n"
]
},
{
diff --git a/nbs/data/TSCategoricalEncoder.joblib b/nbs/data/TSCategoricalEncoder.joblib
index 0aaef0a49..7e1b5a1ac 100644
Binary files a/nbs/data/TSCategoricalEncoder.joblib and b/nbs/data/TSCategoricalEncoder.joblib differ
diff --git a/nbs/data/TSMissingnessEncoder.joblib b/nbs/data/TSMissingnessEncoder.joblib
index c2827f1f8..b60944049 100644
Binary files a/nbs/data/TSMissingnessEncoder.joblib and b/nbs/data/TSMissingnessEncoder.joblib differ
diff --git a/nbs/models/test.pth b/nbs/models/test.pth
index aaa70b1a6..a4262641f 100644
Binary files a/nbs/models/test.pth and b/nbs/models/test.pth differ
diff --git a/tsai/_modidx.py b/tsai/_modidx.py
index d03728542..5466111f6 100644
--- a/tsai/_modidx.py
+++ b/tsai/_modidx.py
@@ -470,6 +470,8 @@
'tsai/data/mixed.py'),
'tsai.data.mixed.MixedDataLoader._split_idxs': ( 'data.mixed.html#mixeddataloader._split_idxs',
'tsai/data/mixed.py'),
+ 'tsai.data.mixed.MixedDataLoader.get_idxs_copy': ( 'data.mixed.html#mixeddataloader.get_idxs_copy',
+ 'tsai/data/mixed.py'),
'tsai.data.mixed.MixedDataLoader.new': ('data.mixed.html#mixeddataloader.new', 'tsai/data/mixed.py'),
'tsai.data.mixed.MixedDataLoader.one_batch': ( 'data.mixed.html#mixeddataloader.one_batch',
'tsai/data/mixed.py'),
diff --git a/tsai/calibration.py b/tsai/calibration.py
index 70f887c47..633a5770a 100644
--- a/tsai/calibration.py
+++ b/tsai/calibration.py
@@ -32,7 +32,7 @@ class TemperatureSetter(nn.Module):
def __init__(self, model, lr=0.01, max_iter=1_000, line_search_fn=None, n_bins=10, verbose=True):
super().__init__()
self.model = ModelWithTemperature(model) if not hasattr(model, 'temperature_scale') else model
- self.lr, self.max_iter, self.line_search_fn, self.n_bins, self.verbose = lr, max_iter, line_search_fn, n_bins, verbose
+ self.lr, self.max_iter, self.line_search_fn, self.n_bins, self.verbose = lr, max_iter, line_search_fn, n_bins, verbose
self.nll_criterion = CrossEntropyLossFlat()
self.ece_criterion = ECELoss(n_bins)
@@ -134,11 +134,11 @@ def plot_calibration_curve(labels, logits, cal_logits=None, figsize=(6,6), n_bin
# %% ../nbs/021_calibration.ipynb 6
@patch
-def calibrate_model(self:Learner, X=None, y=None, lr=1e-2, max_iter=10_000, line_search_fn=None, n_bins=10, strategy='uniform',
+def calibrate_model(self:Learner, X=None, y=None, lr=1e-2, max_iter=10_000, line_search_fn=None, n_bins=10, strategy='uniform',
show_plot=True, figsize=(6,6), verbose=True):
- if X is not None and y is not None:
+ if X is not None and y is not None:
dl = self.dls.valid.new_dl(X, y)
- else:
+ else:
dl = self.dls.valid
assert dl.c == 2, "calibrate_model is only available for binary classification tasks"
temp_setter = TemperatureSetter(self.model, lr=lr, max_iter=max_iter, line_search_fn=line_search_fn, n_bins=n_bins, verbose=verbose)
diff --git a/tsai/data/mixed.py b/tsai/data/mixed.py
index 5281caa04..d04ed60ae 100644
--- a/tsai/data/mixed.py
+++ b/tsai/data/mixed.py
@@ -34,6 +34,8 @@ def __init__(self, *loaders, path='.', shuffle=False, device=None, bs=None):
if hasattr(dl, 'split_idxs'):
self.split_idxs = dl.split_idxs
dl.bs = self.bs
+ if i > 0 and hasattr(dl, 'get_idxs'):
+ dl.get_idxs = self.get_idxs_copy
dl.shuffle_fn = self.shuffle_fn
if self.c is None and hasattr(dl, "c"):
self.c = dl.c
@@ -84,6 +86,9 @@ def _get_idxs(self):
outs += L(b[n_inp:])
self.y_idxs = self._get_vals(outs)
+ def get_idxs_copy(self):
+ return self.loaders[0].get_idxs()
+
def __iter__(self):
z = zip(*[_loaders[i.fake_l.num_workers == 0](i.fake_l)
for i in self.loaders])
diff --git a/tsai/imports.py b/tsai/imports.py
index 6d4f65abb..a8da175aa 100644
--- a/tsai/imports.py
+++ b/tsai/imports.py
@@ -333,6 +333,7 @@ def _has_mps():
def default_device(use=-1):
"Return or set default device; `use_cuda`: -1 - CUDA/mps if available; True - error if not available; False - CPU"
+ # return torch.device("cpu")
if use == -1:
use = defaults.use_cuda
else:
diff --git a/tsai/models/TSSequencerPlus.py b/tsai/models/TSSequencerPlus.py
index 691b04c56..7761fe508 100644
--- a/tsai/models/TSSequencerPlus.py
+++ b/tsai/models/TSSequencerPlus.py
@@ -11,11 +11,11 @@
# %% ../../nbs/069_models.TSSequencerPlus.ipynb 4
class _TSSequencerEncoderLayer(nn.Module):
- def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,
+ def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,
mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):
super().__init__()
self.bilstm = nn.LSTM(q_len, q_len, num_layers=1, bidirectional=True, bias=lstm_bias)
- self.dropout = nn.Dropout(lstm_dropout)
+ self.dropout = nn.Dropout(lstm_dropout) if lstm_dropout else nn.Identity()
self.fc = nn.Linear(2 * q_len, q_len)
self.lstm_norm = nn.LayerNorm(d_model)
self.pwff = PositionwiseFeedForward(d_model, dropout=dropout, act=act, mlp_ratio=mlp_ratio)
@@ -35,7 +35,7 @@ def forward(self, x):
# %% ../../nbs/069_models.TSSequencerPlus.ipynb 5
class _TSSequencerEncoder(nn.Module):
- def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,
+ def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,
mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):
super().__init__()
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
@@ -54,22 +54,22 @@ def forward(self, x):
# %% ../../nbs/069_models.TSSequencerPlus.ipynb 6
class _TSSequencerBackbone(Module):
- def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu',
- lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1,
- pre_norm:bool=False, use_token:bool=True, use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None,
- cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None,
+ def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu',
+ lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1,
+ pre_norm:bool=False, use_token:bool=True, use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None,
+ cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None,
token_size:int=None, tokenizer:Optional[Callable]=None):
# Categorical embeddings
if n_cat_embeds is not None:
n_cat_embeds = listify(n_cat_embeds)
- if cat_embed_dims is None:
+ if cat_embed_dims is None:
cat_embed_dims = [emb_sz_rule(s) for s in n_cat_embeds]
self.to_cat_embed = MultiEmbedding(c_in, n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos)
c_in, seq_len = output_size_calculator(self.to_cat_embed, c_in, seq_len)
else:
self.to_cat_embed = nn.Identity()
-
+
# Sequence embedding
if token_size is not None:
self.tokenizer = SeqTokenizer(c_in, d_model, token_size)
@@ -78,7 +78,7 @@ def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str=
if isinstance(tokenizer, nn.Module): self.tokenizer = tokenizer
else: self.tokenizer = tokenizer(c_in, d_model)
c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)
- else:
+ else:
self.tokenizer = nn.Identity()
# Feature extractor
@@ -88,13 +88,13 @@ def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str=
c_in, seq_len = output_size_calculator(self.feature_extractor, c_in, seq_len)
else:
self.feature_extractor = nn.Identity()
-
+
# Linear projection
if token_size is None and tokenizer is None and feature_extractor is None:
self.linear_proj = nn.Conv1d(c_in, d_model, 1)
else:
self.linear_proj = nn.Identity()
-
+
self.transpose = Transpose(1,2)
# Position embedding & token
@@ -103,10 +103,10 @@ def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str=
self.use_pe = use_pe
self.cls_token = nn.Parameter(torch.zeros(1, 1, d_model))
self.use_token = use_token
- self.emb_dropout = nn.Dropout(dropout)
+ self.emb_dropout = nn.Dropout(dropout) if dropout else nn.Identity()
# Encoder
- self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias,
+ self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias,
lstm_dropout=lstm_dropout, dropout=dropout,
mlp_ratio=mlp_ratio, drop_path_rate=drop_path_rate, act=act, pre_norm=pre_norm)
@@ -114,19 +114,19 @@ def forward(self, x):
# Categorical embeddings
x = self.to_cat_embed(x)
-
+
# Sequence embedding
x = self.tokenizer(x)
# Feature extractor
x = self.feature_extractor(x)
-
+
# Linear projection
x = self.linear_proj(x)
-
+
# Position embedding & token
x = self.transpose(x)
- if self.use_pe:
+ if self.use_pe:
x = x + self.pos_embed
if self.use_token: # token is concatenated after position embedding so that embedding can be learned using self.supervised learning
x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
@@ -134,7 +134,7 @@ def forward(self, x):
# Encoder
x = self.encoder(x)
-
+
# Output
x = x.transpose(1,2)
return x
@@ -154,7 +154,7 @@ class TSSequencerPlus(nn.Sequential):
depth: number of blocks in the encoder.
act: the activation function of positionwise feedforward layer.
lstm_dropout: dropout rate applied to the lstm sublayer.
- dropout: dropout applied to to the embedded sequence steps after position embeddings have been added and
+ dropout: dropout applied to to the embedded sequence steps after position embeddings have been added and
to the mlp sublayer in the encoder.
drop_path_rate: stochastic depth rate.
mlp_ratio: ratio of mlp hidden dim to embedding dim.
@@ -170,38 +170,38 @@ class TSSequencerPlus(nn.Sequential):
cat_pos: list with the position of the categorical variables in the input.
token_size: Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)
tokenizer: nn.Module or callable that will be used to reduce the sequence length
- feature_extractor: nn.Module or callable that will be used to preprocess the time series before
+ feature_extractor: nn.Module or callable that will be used to preprocess the time series before
the embedding step. It is useful to extract features or resample the time series.
- flatten: flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.
+ flatten: flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.
If use_token is False and flatten is False, the model will apply a pooling layer.
- concat_pool: if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.
+ concat_pool: if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.
fc_dropout: dropout applied to the final fully connected layer.
use_bn: flag that indicates if batchnorm will be applied to the head.
bias_init: values used to initialized the output layer.
- y_range: range of possible y values (used in regression tasks).
+ y_range: range of possible y values (used in regression tasks).
custom_head: custom head that will be applied to the network. It must contain all kwargs (pass a partial function)
verbose: flag to control verbosity of the model.
Input:
x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)
"""
-
+
def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',
- lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,
- pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,
+ lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,
+ pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,
cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,
- token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,
- flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,
+ token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,
+ flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,
bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,
**kwargs):
- if use_token and c_out == 1:
+ if use_token and c_out == 1:
use_token = False
pv("use_token set to False as c_out == 1", verbose)
backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,
- lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,
- pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,
- n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,
+ lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,
+ pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,
+ n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,
feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)
self.head_nf = d_model
@@ -215,7 +215,7 @@ def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=
else:
nf = d_model
layers = []
- if use_token:
+ if use_token:
layers += [TokenLayer()]
elif flatten:
layers += [Reshape(-1)]
@@ -225,10 +225,10 @@ def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=
layers = [GACP1d(1) if concat_pool else GAP1d(1)]
if use_bn: layers += [nn.BatchNorm1d(nf)]
if fc_dropout: layers += [nn.Dropout(fc_dropout)]
-
+
# Last layer
linear = nn.Linear(nf, c_out)
- if bias_init is not None:
+ if bias_init is not None:
if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)
else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))
layers += [linear]
@@ -236,6 +236,6 @@ def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=
if y_range: layers += [SigmoidRange(*y_range)]
head = nn.Sequential(*layers)
super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))
-
-
+
+
TSSequencer = TSSequencerPlus
diff --git a/tsai/models/layers.py b/tsai/models/layers.py
index 6baf9f92a..f50833519 100644
--- a/tsai/models/layers.py
+++ b/tsai/models/layers.py
@@ -44,7 +44,7 @@ def test_module_to_torchscript(
verbose:bool=True, # If `True`, prints detailed information about the tracing and scripting process. Defaults to `True`.
):
"Tests if a PyTorch module can be correctly traced or scripted and serialized"
-
+
m = m.eval()
m_name = m.__class__.__name__
@@ -104,16 +104,16 @@ def test_module_to_torchscript(
# %% ../../nbs/029_models.layers.ipynb 6
def init_lin_zero(m):
- if isinstance(m, (nn.Linear)):
+ if isinstance(m, (nn.Linear)):
if getattr(m, 'bias', None) is not None: nn.init.constant_(m.bias, 0)
nn.init.constant_(m.weight, 0)
for l in m.children(): init_lin_zero(l)
-
+
lin_zero_init = init_lin_zero
# %% ../../nbs/029_models.layers.ipynb 7
class SwishBeta(Module):
- def __multiinit__(self, beta=1.):
+ def __multiinit__(self, beta=1.):
self.sigmoid = torch.sigmoid
self.beta = nn.Parameter(torch.Tensor(1).fill_(beta))
def forward(self, x): return x.mul(self.sigmoid(x*self.beta))
@@ -122,7 +122,7 @@ def forward(self, x): return x.mul(self.sigmoid(x*self.beta))
class SmeLU(nn.Module):
"Smooth ReLU activation function based on https://arxiv.org/pdf/2202.06499.pdf"
- def __init__(self,
+ def __init__(self,
beta: float = 2. # Beta value
) -> None:
super().__init__()
@@ -136,7 +136,7 @@ class Chomp1d(nn.Module):
def __init__(self, chomp_size):
super(Chomp1d, self).__init__()
self.chomp_size = chomp_size
-
+
def forward(self, x):
return x[:, :, :-self.chomp_size].contiguous()
@@ -151,7 +151,7 @@ class Pad1d(nn.ConstantPad1d):
def __init__(self, padding, value=0.):
super().__init__(padding, value)
-
+
# @delegates(nn.Conv1d.__init__)
class SameConv1d(Module):
"Conv1d with padding='same'"
@@ -198,15 +198,15 @@ def __init__(self, ni, nf, ks=(3, 3), stride=(1, 1), dilation=(1, 1), **kwargs):
def forward(self, x):
self.padding = same_padding2d(x.shape[-2], x.shape[-1], self.ks, dilation=self.dilation) #stride=self.stride not used in padding calculation!
return self.conv2d_same(self.pad(self.padding)(x))
-
-
+
+
# @delegates(nn.Conv2d.__init__)
def Conv2d(ni, nf, kernel_size=None, ks=None, stride=1, padding='same', dilation=1, init='auto', bias_std=0.01, **kwargs):
"conv1d layer with padding='same', 'valid', or any integer (defaults to 'same')"
assert not (kernel_size and ks), 'use kernel_size or ks but not both simultaneously'
assert kernel_size is not None or ks is not None, 'you need to pass a ks'
kernel_size = kernel_size or ks
- if padding == 'same':
+ if padding == 'same':
conv = Conv2dSame(ni, nf, kernel_size, stride=stride, dilation=dilation, **kwargs)
elif padding == 'valid': conv = nn.Conv2d(ni, nf, kernel_size, stride=stride, padding=0, dilation=dilation, **kwargs)
else: conv = nn.Conv2d(ni, nf, kernel_size, stride=stride, padding=padding, dilation=dilation, **kwargs)
@@ -228,8 +228,8 @@ def Conv1d(ni, nf, kernel_size=None, ks=None, stride=1, padding='same', dilation
assert not (kernel_size and ks), 'use kernel_size or ks but not both simultaneously'
assert kernel_size is not None or ks is not None, 'you need to pass a ks'
kernel_size = kernel_size or ks
- if padding == 'same':
- if kernel_size%2==1:
+ if padding == 'same':
+ if kernel_size%2==1:
conv = nn.Conv1d(ni, nf, kernel_size, stride=stride, padding=kernel_size//2 * dilation, dilation=dilation, **kwargs)
else:
conv = SameConv1d(ni, nf, kernel_size, stride=stride, dilation=dilation, **kwargs)
@@ -245,10 +245,10 @@ def __init__(self, ni, nf, ks, stride=1, padding='same', dilation=1, bias=True,
self.depthwise_conv = Conv1d(ni, ni, ks, stride=stride, padding=padding, dilation=dilation, groups=ni, bias=bias)
self.pointwise_conv = nn.Conv1d(ni, nf, 1, stride=1, padding=0, dilation=1, groups=1, bias=bias)
if bias:
- if bias_std != 0:
+ if bias_std != 0:
normal_(self.depthwise_conv.bias, 0, bias_std)
normal_(self.pointwise_conv.bias, 0, bias_std)
- else:
+ else:
self.depthwise_conv.bias.data.zero_()
self.pointwise_conv.bias.data.zero_()
@@ -286,7 +286,7 @@ def __init__(self, ni, nf, kernel_size=None, ks=3, stride=1, padding='same', bia
if norm_type==NormType.Weight: conv = weight_norm(conv)
elif norm_type==NormType.Spectral: conv = spectral_norm(conv)
layers += [conv]
- act_bn = []
+ act_bn = []
if act is not None: act_bn.append(act)
if bn: act_bn.append(BatchNorm(nf, norm_type=norm_type, ndim=ndim))
if inn: act_bn.append(InstanceNorm(nf, norm_type=norm_type, ndim=ndim))
@@ -294,8 +294,8 @@ def __init__(self, ni, nf, kernel_size=None, ks=3, stride=1, padding='same', bia
if dropout: layers += [nn.Dropout(dropout)]
layers += act_bn
if xtra: layers.append(xtra)
- super().__init__(*layers)
-
+ super().__init__(*layers)
+
Conv = named_partial('Conv', ConvBlock, norm=None, act=None)
ConvBN = named_partial('ConvBN', ConvBlock, norm='Batch', act=None)
CoordConv = named_partial('CoordConv', ConvBlock, norm=None, act=None, coord=True)
@@ -335,7 +335,7 @@ def SEModule1d(ni, reduction=16, act=nn.ReLU, act_kwargs={}):
"Squeeze and excitation module for 1d"
nf = math.ceil(ni//reduction/8)*8
assert nf != 0, 'nf cannot be 0'
- return SequentialEx(nn.AdaptiveAvgPool1d(1),
+ return SequentialEx(nn.AdaptiveAvgPool1d(1),
ConvBlock(ni, nf, ks=1, norm=None, act=act, act_kwargs=act_kwargs),
ConvBlock(nf, ni, ks=1, norm=None, act=nn.Sigmoid), ProdLayer())
@@ -396,88 +396,89 @@ class Unfold(Module):
def __init__(self, dim, size, step=1): self.dim, self.size, self.step = dim, size, step
def forward(self, x:Tensor) -> Tensor: return x.unfold(dimension=self.dim, size=self.size, step=self.step)
def __repr__(self): return f"{self.__class__.__name__}(dim={self.dim}, size={self.size}, step={self.step})"
-
-
+
+
class Permute(Module):
def __init__(self, *dims): self.dims = dims
def forward(self, x:Tensor) -> Tensor: return x.permute(self.dims)
def __repr__(self): return f"{self.__class__.__name__}(dims={', '.join([str(d) for d in self.dims])})"
-
-
+
+
class Transpose(Module):
- def __init__(self, *dims, contiguous=False): self.dims, self.contiguous = dims, contiguous
- def forward(self, x):
+ def __init__(self, *dims, contiguous=True): self.dims, self.contiguous = dims, contiguous
+ def forward(self, x):
+ x = x.contiguous()
if self.contiguous: return x.transpose(*self.dims).contiguous()
else: return x.transpose(*self.dims)
- def __repr__(self):
+ def __repr__(self):
if self.contiguous: return f"{self.__class__.__name__}(dims={', '.join([str(d) for d in self.dims])}).contiguous()"
else: return f"{self.__class__.__name__}({', '.join([str(d) for d in self.dims])})"
-
-
+
+
class View(Module):
def __init__(self, *shape): self.shape = shape
- def forward(self, x):
+ def forward(self, x):
return x.view(x.shape[0], -1).contiguous() if not self.shape else x.view(-1).contiguous() if self.shape == (-1,) else \
x.view(x.shape[0], *self.shape).contiguous()
def __repr__(self): return f"{self.__class__.__name__}({', '.join(['bs'] + [str(s) for s in self.shape])})"
-
-
+
+
class Reshape(Module):
def __init__(self, *shape): self.shape = shape
def forward(self, x):
- return x.reshape(x.shape[0], -1) if not self.shape else x.reshape(-1) if self.shape == (-1,) else x.reshape(x.shape[0], *self.shape)
+ return x.contiguous().reshape(x.shape[0], -1) if not self.shape else x.contiguous().reshape(-1) if self.shape == (-1,) else x.contiguous().reshape(x.shape[0], *self.shape)
def __repr__(self): return f"{self.__class__.__name__}({', '.join(['bs'] + [str(s) for s in self.shape])})"
-
-
+
+
class Max(Module):
def __init__(self, dim=None, keepdim=False): self.dim, self.keepdim = dim, keepdim
def forward(self, x): return x.max(self.dim, keepdim=self.keepdim)[0]
def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim}, keepdim={self.keepdim})'
-
+
class LastStep(Module):
def forward(self, x): return x[..., -1]
def __repr__(self): return f'{self.__class__.__name__}()'
-
-
+
+
class SoftMax(Module):
"SoftMax layer"
def __init__(self, dim=-1):
self.dim = dim
def forward(self, x):
return F.softmax(x, dim=self.dim)
- def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'
-
+ def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'
+
class Clamp(Module):
def __init__(self, min=None, max=None):
self.min, self.max = min, max
def forward(self, x):
return x.clamp(min=self.min, max=self.max)
- def __repr__(self): return f'{self.__class__.__name__}(min={self.min}, max={self.max})'
-
-
+ def __repr__(self): return f'{self.__class__.__name__}(min={self.min}, max={self.max})'
+
+
class Clip(Module):
def __init__(self, min=None, max=None):
self.min, self.max = min, max
-
+
def forward(self, x):
if self.min is not None:
x = torch.maximum(x, self.min)
if self.max is not None:
x = torch.minimum(x, self.max)
return x
- def __repr__(self): return f'{self.__class__.__name__}()'
-
-
+ def __repr__(self): return f'{self.__class__.__name__}()'
+
+
class ReZero(Module):
def __init__(self, module):
self.module = module
self.alpha = nn.Parameter(torch.zeros(1))
def forward(self, x):
return x + self.alpha * self.module(x)
-
-
+
+
Noop = nn.Sequential()
# %% ../../nbs/029_models.layers.ipynb 38
@@ -514,7 +515,7 @@ def forward(self, x):
class Sequential(nn.Sequential):
"""Class that allows you to pass one or multiple inputs"""
def forward(self, *x):
- for i, module in enumerate(self._modules.values()):
+ for i, module in enumerate(self._modules.values()):
x = module(*x) if isinstance(x, (list, tuple, L)) else module(x)
return x
@@ -531,15 +532,15 @@ def forward(self, x):
return self.module(x)
# Squash samples and timesteps into a single axis
- x_reshape = x.contiguous().view(-1, x.size(-1)) # (samples * timesteps, input_size)
+ x_reshape = x.contiguous().reshape(-1, x.size(-1)) # (samples * timesteps, input_size)
y = self.module(x_reshape)
# We have to reshape Y
if self.batch_first:
- y = y.contiguous().view(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size)
+ y = y.contiguous().reshape(x.size(0), -1, y.size(-1)) # (samples, timesteps, output_size)
else:
- y = y.view(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size)
+ y = y.reshape(-1, x.size(1), y.size(-1)) # (timesteps, samples, output_size)
return y
@@ -581,8 +582,8 @@ def __init__(self, n_classes=1, dirichlet=False):
def forward(self, x):
if self.log_softmax: x = F.log_softmax(x, dim=-1)
return self.ms(x)
-
-
+
+
def get_calibrator(calibrator=None, n_classes=1, **kwargs):
if calibrator is None or not calibrator: return noop
elif calibrator.lower() == 'temp': return Temp_Scale(dirichlet=False, **kwargs)
@@ -600,27 +601,27 @@ def __init__(self, class_priors):
self.class_priors = class_priors
def forward(self, x):
return x.add(self.class_priors)
-
+
LogitAdjLayer = LogitAdjustmentLayer
# %% ../../nbs/029_models.layers.ipynb 51
class PPV(Module):
- def __init__(self, dim=-1):
+ def __init__(self, dim=-1):
self.dim = dim
- def forward(self, x):
+ def forward(self, x):
return torch.gt(x, 0).sum(dim=self.dim).float() / x.shape[self.dim]
def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'
-
+
class PPAuc(Module):
- def __init__(self, dim=-1):
+ def __init__(self, dim=-1):
self.dim = dim
- def forward(self, x):
+ def forward(self, x):
x = F.relu(x).sum(self.dim) / (abs(x).sum(self.dim) + 1e-8)
return x
def __repr__(self): return f'{self.__class__.__name__}(dim={self.dim})'
-
-
+
+
class MaxPPVPool1d(Module):
"Drop-in replacement for AdaptiveConcatPool1d - multiplies nf by 2"
def forward(self, x):
@@ -631,8 +632,8 @@ def forward(self, x):
# %% ../../nbs/029_models.layers.ipynb 53
class AdaptiveWeightedAvgPool1d(Module):
'''Global Pooling layer that performs a weighted average along the temporal axis
-
- It can be considered as a channel-wise form of local temporal attention. Inspired by the paper:
+
+ It can be considered as a channel-wise form of local temporal attention. Inspired by the paper:
Hyun, J., Seong, H., & Kim, E. (2019). Universal Pooling--A New Pooling Method for Convolutional Neural Networks. arXiv preprint arXiv:1907.11440.'''
def __init__(self, n_in, seq_len, mult=2, n_layers=2, ln=False, dropout=0.5, act=nn.ReLU(), zero_init=True):
@@ -641,7 +642,7 @@ def __init__(self, n_in, seq_len, mult=2, n_layers=2, ln=False, dropout=0.5, act
inp_mult = mult if i > 0 else 1
out_mult = mult if i < n_layers -1 else 1
p = dropout[i] if is_listy(dropout) else dropout
- layers.append(LinLnDrop(seq_len * inp_mult, seq_len * out_mult, ln=False, p=p,
+ layers.append(LinLnDrop(seq_len * inp_mult, seq_len * out_mult, ln=False, p=p,
act=act if i < n_layers-1 and n_layers > 1 else None))
self.layers = layers
self.softmax = SoftMax(-1)
@@ -661,8 +662,8 @@ def __init__(self, output_size=1):
self.flatten = Reshape()
def forward(self, x):
return self.flatten(self.gap(x))
-
-
+
+
class GACP1d(Module):
"Global AdaptiveConcatPool + Flatten"
def __init__(self, output_size=1):
@@ -670,7 +671,7 @@ def __init__(self, output_size=1):
self.flatten = Reshape()
def forward(self, x):
return self.flatten(self.gacp(x))
-
+
class GAWP1d(Module):
"Global AdaptiveWeightedAvgPool1d + Flatten"
@@ -682,13 +683,13 @@ def forward(self, x):
# %% ../../nbs/029_models.layers.ipynb 55
class GlobalWeightedAveragePool1d(Module):
- """ Global Weighted Average Pooling layer
-
+ """ Global Weighted Average Pooling layer
+
Inspired by Building Efficient CNN Architecture for Offline Handwritten Chinese Character Recognition
https://arxiv.org/pdf/1804.01259.pdf
"""
-
- def __init__(self, n_in, seq_len):
+
+ def __init__(self, n_in, seq_len):
self.weight = nn.Parameter(torch.ones(1, n_in, seq_len))
self.bias = nn.Parameter(torch.zeros(1, n_in, seq_len))
@@ -704,26 +705,26 @@ def gwa_pool_head(n_in, c_out, seq_len, bn=True, fc_dropout=0.):
# %% ../../nbs/029_models.layers.ipynb 57
class AttentionalPool1d(Module):
"""Global Adaptive Pooling layer inspired by Attentional Pooling for Action Recognition https://arxiv.org/abs/1711.01467"""
- def __init__(self, n_in, c_out, bn=False):
+ def __init__(self, n_in, c_out, bn=False):
store_attr()
self.bn = nn.BatchNorm1d(n_in) if bn else None
self.conv1 = Conv1d(n_in, 1, 1)
self.conv2 = Conv1d(n_in, c_out, 1)
def forward(self, x):
- if self.bn is not None: x = self.bn(x)
+ if self.bn is not None: x = self.bn(x)
return (self.conv1(x) @ self.conv2(x).transpose(1,2)).transpose(1,2)
-
+
class GAttP1d(nn.Sequential):
def __init__(self, n_in, c_out, bn=False):
super().__init__(AttentionalPool1d(n_in, c_out, bn=bn), Reshape())
-
+
def attentional_pool_head(n_in, c_out, seq_len=None, bn=True, **kwargs):
return nn.Sequential(AttentionalPool1d(n_in, c_out, bn=bn, **kwargs), Reshape())
# %% ../../nbs/029_models.layers.ipynb 60
class PoolingLayer(Module):
- def __init__(self, method='cls', seq_len=None, token=True, seq_last=True):
+ def __init__(self, method='cls', seq_len=None, token=True, seq_last=True):
method = method.lower()
assert method in ['cls', 'max', 'mean', 'max-mean', 'linear', 'conv1d', 'flatten']
if method == 'cls': assert token, 'you can only choose method=cls if a token exists'
@@ -733,11 +734,11 @@ def __init__(self, method='cls', seq_len=None, token=True, seq_last=True):
if method == 'linear' or method == 'conv1d':
self.linear = nn.Linear(seq_len - token, 1)
- def forward(self, x):
+ def forward(self, x):
if self.method == 'cls':
return x[..., 0] if self.seq_last else x[:, 0]
if self.token:
- x = x[..., 1:] if self.seq_last else x[:, 1:]
+ x = x[..., 1:] if self.seq_last else x[:, 1:]
if self.method == 'max':
return torch.max(x, -1)[0] if self.seq_last else torch.max(x, 1)[0]
elif self.method == 'mean':
@@ -749,7 +750,7 @@ def forward(self, x):
return x.flatten(1)
elif self.method == 'linear' or self.method == 'conv1d':
return self.linear(x)[...,0] if self.seq_last else self.linear(x.transpose(1,2))[...,0]
-
+
def __repr__(self): return f"{self.__class__.__name__}(method={self.method}, token={self.token}, seq_last={self.seq_last})"
# %% ../../nbs/029_models.layers.ipynb 63
@@ -779,8 +780,8 @@ def get_act_fn(act, **act_kwargs):
class RevIN(nn.Module):
""" Reversible Instance Normalization layer adapted from
- Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).
- Reversible instance normalization for accurate time-series forecasting against distribution shift.
+ Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).
+ Reversible instance normalization for accurate time-series forecasting against distribution shift.
In International Conference on Learning Representations.
Original code: https://github.com/ts-kim/RevIN
"""
@@ -797,20 +798,20 @@ def __init__(self,
if self.affine:
self.weight = nn.Parameter(torch.ones(1, c_in, 1))
self.bias = nn.Parameter(torch.zeros(1, c_in, 1))
-
+
def forward(self, x:Tensor, mode:Tensor):
"""Args:
x: rank 3 tensor with shape [batch size x c_in x sequence length]
mode: torch.tensor(True) to normalize data and torch.tensor(False) to reverse normalization
"""
-
+
# Normalize
if mode: return self.normalize(x)
-
+
# Denormalize
else: return self.denormalize(x)
-
+
def normalize(self, x):
if self.subtract_last:
self.sub = x[..., -1].unsqueeze(-1).detach()
@@ -827,7 +828,7 @@ def normalize(self, x):
x = x.sub(self.sub)
x = x.div(self.std)
return x
-
+
def denormalize(self, x):
if self.affine:
x = x.sub(self.bias)
@@ -844,8 +845,8 @@ def denormalize(self, x):
class RevIN(nn.Module):
""" Reversible Instance Normalization layer adapted from
- Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).
- Reversible instance normalization for accurate time-series forecasting against distribution shift.
+ Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2021, September).
+ Reversible instance normalization for accurate time-series forecasting against distribution shift.
In International Conference on Learning Representations.
Original code: https://github.com/ts-kim/RevIN
"""
@@ -862,16 +863,16 @@ def __init__(self,
self.weight = nn.Parameter(torch.ones(1, c_in, 1))
self.bias = nn.Parameter(torch.zeros(1, c_in, 1))
self.sub, self.std, self.mul, self.add = torch.zeros(1), torch.ones(1), torch.ones(1), torch.zeros(1)
-
+
def forward(self, x:Tensor, mode:Tensor):
"""Args:
x: rank 3 tensor with shape [batch size x c_in x sequence length]
mode: torch.tensor(True) to normalize data and torch.tensor(False) to reverse normalization
"""
-
+
# Normalize
- if mode:
+ if mode:
if self.subtract_last:
self.sub = x[..., -1].unsqueeze(-1).detach()
else:
@@ -887,9 +888,9 @@ def forward(self, x:Tensor, mode:Tensor):
x = x.sub(self.sub)
x = x.div(self.std)
return x
-
+
# Denormalize
- else:
+ else:
if self.affine:
x = x.sub(self.bias)
x = x.div(self.weight)
@@ -951,8 +952,8 @@ def create_conv_head(*args, adaptive_size=None, y_range=None):
c_out = args[1]
layers = [nn.AdaptiveAvgPool1d(adaptive_size)] if adaptive_size is not None else []
for i in range(2):
- if nf > 1:
- layers += [ConvBlock(nf, nf // 2, 1)]
+ if nf > 1:
+ layers += [ConvBlock(nf, nf // 2, 1)]
nf = nf//2
else: break
layers += [ConvBlock(nf, c_out, 1), GAP1d(1)]
@@ -998,7 +999,7 @@ def create_rnn_head(*args, fc_dropout=0., bn=False, y_range=None):
# %% ../../nbs/029_models.layers.ipynb 84
def imputation_head(c_in, c_out, seq_len=None, ks=1, y_range=None, fc_dropout=0.):
layers = [nn.Dropout(fc_dropout), nn.Conv1d(c_in, c_out, ks)]
- if y_range is not None:
+ if y_range is not None:
y_range = (tensor(y_range[0]), tensor(y_range[1]))
layers += [SigmoidRange(*y_range)]
return nn.Sequential(*layers)
@@ -1017,10 +1018,10 @@ def __init__(self, n_in, n_out, seq_len, d, conv_first=True, conv_bn=False, lin_
fd *= _d
shape.append(_d)
if n_out > 1: shape.append(n_out)
- else:
+ else:
fd = d
shape = [d, n_out] if n_out > 1 else [d]
-
+
conv = [BatchNorm(n_in, ndim=1)] if conv_bn else []
conv.append(Conv1d(n_in, n_out, 1, padding=0, bias=not conv_bn, **kwargs))
l = [Transpose(-1, -2), BatchNorm(seq_len, ndim=1), Transpose(-1, -2)] if lin_bn else []
@@ -1032,7 +1033,7 @@ def __init__(self, n_in, n_out, seq_len, d, conv_first=True, conv_bn=False, lin_
layers += [Reshape(*shape)]
super().__init__(*layers)
-
+
conv_lin_nd_head = create_conv_lin_nd_head
conv_lin_3d_head = create_conv_lin_nd_head # included for compatibility
create_conv_lin_3d_head = create_conv_lin_nd_head # included for compatibility
@@ -1055,10 +1056,10 @@ def __init__(self, n_in, n_out, seq_len=None, d=None, flatten=False, use_bn=Fals
fd *= _d
shape.append(_d)
if n_out > 1: shape.append(n_out)
- else:
+ else:
fd = d
shape = [d, n_out] if n_out > 1 else [d]
-
+
layers = []
if use_bn:
layers += [nn.BatchNorm1d(n_in)]
@@ -1083,7 +1084,7 @@ def __init__(self, n_in, n_out, seq_len=None, d=None, flatten=False, use_bn=Fals
layers += [Reshape(*shape)]
super().__init__(*layers)
-
+
create_lin_nd_head = lin_nd_head
lin_3d_head = lin_nd_head # included for backwards compatiblity
create_lin_3d_head = lin_nd_head # included for backwards compatiblity
@@ -1104,7 +1105,7 @@ def __init__(self, n_in, n_out, seq_len=None, d=None, use_bn=False, fc_dropout=0
fd *= _d
shape.append(_d)
if n_out > 1: shape.append(n_out)
- else:
+ else:
fd = d
shape = [d, n_out] if n_out > 1 else [d]
@@ -1141,7 +1142,7 @@ def __init__(self, n_in, n_out, seq_len=None, d=None, use_bn=False, fc_dropout=0
fd *= _d
shape.append(_d)
if n_out > 1: shape.append(n_out)
- else:
+ else:
fd = d
shape = [d, n_out] if n_out > 1 else [d]
@@ -1172,17 +1173,17 @@ def __init__(self, n_in, n_out, seq_len, d, use_bn=False, **kwargs):
layers += [Conv(n_in, n_out, 1, **kwargs), Transpose(-1,-2)]
if n_out == 1: layers += [Squeeze(-1)]
super().__init__(*layers)
-
+
conv_3d_head = create_conv_3d_head
# %% ../../nbs/029_models.layers.ipynb 104
def universal_pool_head(n_in, c_out, seq_len, mult=2, pool_n_layers=2, pool_ln=True, pool_dropout=0.5, pool_act=nn.ReLU(),
zero_init=True, bn=True, fc_dropout=0.):
- return nn.Sequential(AdaptiveWeightedAvgPool1d(n_in, seq_len, n_layers=pool_n_layers, mult=mult, ln=pool_ln, dropout=pool_dropout, act=pool_act),
+ return nn.Sequential(AdaptiveWeightedAvgPool1d(n_in, seq_len, n_layers=pool_n_layers, mult=mult, ln=pool_ln, dropout=pool_dropout, act=pool_act),
Reshape(), LinBnDrop(n_in, c_out, p=fc_dropout, bn=bn))
# %% ../../nbs/029_models.layers.ipynb 106
-heads = [mlp_head, fc_head, average_pool_head, max_pool_head, concat_pool_head, pool_plus_head, conv_head, rnn_head,
+heads = [mlp_head, fc_head, average_pool_head, max_pool_head, concat_pool_head, pool_plus_head, conv_head, rnn_head,
conv_lin_nd_head, lin_nd_head, conv_3d_head, attentional_pool_head, universal_pool_head, gwa_pool_head]
# %% ../../nbs/029_models.layers.ipynb 108
@@ -1219,7 +1220,7 @@ def forward(self, x):
scale = self.sigma * (x.detach() if self.is_relative_detach else x)
sampled_noise = torch.empty(x.size(), device=x.device).normal_() * scale
x = x + sampled_noise
- return x
+ return x
# %% ../../nbs/029_models.layers.ipynb 114
class PositionwiseFeedForward(nn.Sequential):
@@ -1238,11 +1239,11 @@ def __repr__(self): return f"{self.__class__.__name__}()"
# %% ../../nbs/029_models.layers.ipynb 116
class ScaledDotProductAttention(Module):
- r"""Scaled Dot-Product Attention module (Attention is all you need by Vaswani et al., 2017) with optional residual attention from previous layer
- (Realformer: Transformer likes residual attention by He et al, 2020) and locality self sttention (Vision Transformer for Small-Size Datasets
+ r"""Scaled Dot-Product Attention module (Attention is all you need by Vaswani et al., 2017) with optional residual attention from previous layer
+ (Realformer: Transformer likes residual attention by He et al, 2020) and locality self sttention (Vision Transformer for Small-Size Datasets
by Lee et al, 2021)"""
- def __init__(self, d_model, n_heads, attn_dropout=0., res_attention=False, lsa=False):
+ def __init__(self, d_model, n_heads, attn_dropout=0., res_attention=False, lsa=False):
self.attn_dropout = nn.Dropout(attn_dropout)
self.res_attention = res_attention
head_dim = d_model // n_heads
@@ -1259,7 +1260,7 @@ def forward(self, q:Tensor, k:Tensor, v:Tensor, prev:Optional[Tensor]=None, key_
key_padding_mask: [bs x seq_len]
attn_mask : [1 x seq_len x seq_len]
- Output shape:
+ Output shape:
output: [bs x n_heads x q_len x d_v]
attn : [bs x n_heads x q_len x seq_len]
scores : [bs x n_heads x q_len x seq_len]
@@ -1269,7 +1270,7 @@ def forward(self, q:Tensor, k:Tensor, v:Tensor, prev:Optional[Tensor]=None, key_
attn_scores = torch.matmul(q, k) * self.scale # attn_scores : [bs x n_heads x max_q_len x q_len]
# Add pre-softmax attention scores from the previous layer (optional)
- if prev is not None: attn_scores = attn_scores + prev
+ if prev is not None: attn_scores = attn_scores + prev
# Attention mask (optional)
if attn_mask is not None: # attn_mask with shape [q_len x seq_len] - only used when q_len == seq_len
@@ -1328,9 +1329,9 @@ def forward(self, Q:Tensor, K:Optional[Tensor]=None, V:Optional[Tensor]=None, pr
if V is None: V = Q
# Linear (+ split in multiple heads)
- q_s = self.W_Q(Q).view(bs, -1, self.n_heads, self.d_k).transpose(1,2) # q_s : [bs x n_heads x max_q_len x d_k]
- k_s = self.W_K(K).view(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1) # k_s : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)
- v_s = self.W_V(V).view(bs, -1, self.n_heads, self.d_v).transpose(1,2) # v_s : [bs x n_heads x q_len x d_v]
+ q_s = self.W_Q(Q).reshape(bs, -1, self.n_heads, self.d_k).transpose(1,2) # q_s : [bs x n_heads x max_q_len x d_k]
+ k_s = self.W_K(K).reshape(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1) # k_s : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)
+ v_s = self.W_V(V).reshape(bs, -1, self.n_heads, self.d_v).transpose(1,2) # v_s : [bs x n_heads x q_len x d_v]
# Apply Scaled Dot-Product Attention (multiple heads)
if self.res_attention:
@@ -1340,11 +1341,11 @@ def forward(self, Q:Tensor, K:Optional[Tensor]=None, V:Optional[Tensor]=None, pr
# output: [bs x n_heads x q_len x d_v], attn: [bs x n_heads x q_len x q_len], scores: [bs x n_heads x max_q_len x q_len]
# back to the original inputs dimensions
- output = output.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * self.d_v) # output: [bs x q_len x n_heads * d_v]
+ output = output.transpose(1, 2).contiguous().reshape(bs, -1, self.n_heads * self.d_v) # output: [bs x q_len x n_heads * d_v]
output = self.to_out(output)
if self.res_attention: return output, attn_weights, attn_scores
- else: return output, attn_weights
+ else: return output, attn_weights
# %% ../../nbs/029_models.layers.ipynb 125
class MultiConv1d(Module):
@@ -1399,18 +1400,18 @@ def __init__(self, c_in, n_cat_embeds, cat_embed_dims=None, cat_pos=None, std=0.
cat_n_embeds = listify(n_cat_embeds)
if cat_padding_idxs is None: cat_padding_idxs = [None]
else: cat_padding_idxs = listify(cat_padding_idxs)
- if len(cat_padding_idxs) == 1 and len(cat_padding_idxs) < len(cat_n_embeds):
+ if len(cat_padding_idxs) == 1 and len(cat_padding_idxs) < len(cat_n_embeds):
cat_padding_idxs = cat_padding_idxs * len(cat_n_embeds)
assert len(cat_n_embeds) == len(cat_padding_idxs)
- if cat_embed_dims is None:
+ if cat_embed_dims is None:
cat_embed_dims = [emb_sz_rule(s) for s in cat_n_embeds]
else:
cat_embed_dims = listify(cat_embed_dims)
if len(cat_embed_dims) == 1: cat_embed_dims = cat_embed_dims * len(cat_n_embeds)
assert len(cat_embed_dims) == len(cat_n_embeds)
- if cat_pos:
- cat_pos = torch.as_tensor(listify(cat_pos))
- else:
+ if cat_pos:
+ cat_pos = torch.as_tensor(listify(cat_pos))
+ else:
cat_pos = torch.arange(len(cat_n_embeds))
self.register_buffer("cat_pos", cat_pos)
cont_pos = torch.tensor([p for p in torch.arange(c_in) if p not in self.cat_pos])