training_transformer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/subdiduntil2/AVLSI_ECE_NTUA/blob/main/training_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XqpTo-6scgrc",
        "outputId": "0879105c-2f27-48f6-8969-94ca939add19"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torch-geometric\n",
            "  Downloading torch_geometric-2.3.1.tar.gz (661 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m661.6/661.6 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (4.66.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.23.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.10.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (3.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (2.31.0)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (3.1.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.2.2)\n",
            "Requirement already satisfied: psutil>=5.8.0 in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (5.9.5)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch-geometric) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (2023.7.22)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->torch-geometric) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->torch-geometric) (3.2.0)\n",
            "Building wheels for collected packages: torch-geometric\n",
            "  Building wheel for torch-geometric (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for torch-geometric: filename=torch_geometric-2.3.1-py3-none-any.whl size=910454 sha256=61fbf93a9c48a0b3d658fffdbd23b73a06ad6e1f9a14111fb3b860f915c64d39\n",
            "  Stored in directory: /root/.cache/pip/wheels/ac/dc/30/e2874821ff308ee67dcd7a66dbde912411e19e35a1addda028\n",
            "Successfully built torch-geometric\n",
            "Installing collected packages: torch-geometric\n",
            "Successfully installed torch-geometric-2.3.1\n",
            "Requirement already satisfied: torch-geometric in /usr/local/lib/python3.10/dist-packages (2.3.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (4.66.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.23.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.10.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (3.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (2.31.0)\n",
            "Requirement already satisfied: pyparsing in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (3.1.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (1.2.2)\n",
            "Requirement already satisfied: psutil>=5.8.0 in /usr/local/lib/python3.10/dist-packages (from torch-geometric) (5.9.5)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch-geometric) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torch-geometric) (2023.7.22)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->torch-geometric) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->torch-geometric) (3.2.0)\n"
          ]
        }
      ],
      "source": [
        "!pip3 install torch-geometric\n",
        "!pip install torch-geometric"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BLbP2-NYbmj1",
        "outputId": "5239a616-bd3c-4bf6-c594-f36825e4dafa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "libraries loaded\n"
          ]
        }
      ],
      "source": [
        "#models and pipeline dependencies\n",
        "import os\n",
        "import warnings\n",
        "import numpy as np\n",
        "import torch\n",
        "import math\n",
        "from torch_geometric.data import Data\n",
        "from torch_geometric.loader import DataLoader\n",
        "from torch_geometric.nn import Set2Set, AttentionalAggregation, MeanAggregation, GATConv\n",
        "from torch_geometric.nn.aggr import Aggregation\n",
        "from torch_geometric.nn.inits import reset\n",
        "from torch_geometric.utils import softmax\n",
        "from typing import Optional\n",
        "from torch import Tensor\n",
        "# from models import *\n",
        "from torchsummary import summary\n",
        "from prettytable import PrettyTable\n",
        "from typing import Tuple\n",
        "import torch.nn as nn\n",
        "from torch.nn.functional import softmax\n",
        "import torch_geometric\n",
        "from torch.functional import F\n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "from math import log2\n",
        "drive.mount('/content/drive')\n",
        "print(\"libraries loaded\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#functions and outside params\n",
        "\n",
        "def count_parameters(model):\n",
        "    table = PrettyTable([\"Modules\", \"Parameters\"])\n",
        "    total_params = 0\n",
        "    for name, parameter in model.named_parameters():\n",
        "        if not parameter.requires_grad: continue\n",
        "        params = parameter.numel()\n",
        "        table.add_row([name, params])\n",
        "        total_params+=params\n",
        "    print(table)\n",
        "    print(f\"Total Trainable Params: {total_params}\")\n",
        "    return total_params\n",
        "\n",
        "num_node_feats = 11\n",
        "num_edge_feats = 2\n",
        "num_edges = 6000\n",
        "num_nodes = 3000\n",
        "num_graphs = 20\n",
        "num_embs = 1000 #tbc\n",
        "emb_dim = 100\n",
        "batch_size = 1\n",
        "epochs = 6\n",
        "loss_all = 0.0\n",
        "max_split_size_mb = 200\n",
        "data_list = []\n",
        "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(\"device is -> \",DEVICE)\n",
        "print(\"loaded params and functions\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wXh5wcGpH8FA",
        "outputId": "290d31f1-0f5d-4f25-ecb7-8d7ca8b94499"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "device is ->  cuda\n",
            "loaded params and functions\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cC9ppC_CcQef",
        "outputId": "2ae76909-31e8-4641-8489-3315f468d04d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "loading of models done\n"
          ]
        }
      ],
      "source": [
        "#model class definitions\n",
        "class AttentionalAggregationCustom(Aggregation):\n",
        "    def __init__(self, gate_nn: torch.nn.Module,nn: Optional[torch.nn.Module] = None):\n",
        "        super().__init__()\n",
        "        self.gate_nn = gate_nn\n",
        "        self.nn = nn\n",
        "        # self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        reset(self.gate_nn)\n",
        "        reset(self.nn)\n",
        "\n",
        "    def forward(self, x: Tensor, index: Optional[Tensor] = None, ptr: Optional[Tensor] = None, dim_size: Optional[int] = None, dim: int = -2) -> Tensor:\n",
        "        self.assert_two_dimensional_input(x, dim)\n",
        "        gate = self.gate_nn(x)\n",
        "        x = self.nn(x) if self.nn is not None else x\n",
        "        print(\"gate gives -> \",gate,type(gate))\n",
        "        print(\"x is -> \",x,np.shape(x))\n",
        "        gate = softmax(gate, index, ptr, dim_size, dim)\n",
        "        return self.reduce(gate * x, index, ptr, dim_size, dim)\n",
        "\n",
        "    def __repr__(self) -> str:\n",
        "        return (f'{self.__class__.__name__}(gate_nn={self.gate_nn}, '\n",
        "                f'nn={self.nn})')\n",
        "\n",
        "class FeedForward(nn.Module):\n",
        "    \"\"\" a simple linear layer followed by a non-linearity \"\"\"\n",
        "    def __init__(self, in_channels, dropout=0.0):\n",
        "        super().__init__()\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Linear(in_channels, 4*in_channels),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(4*in_channels, in_channels),\n",
        "            nn.Dropout(dropout),\n",
        "        )\n",
        "        self.init_weights(FeedForward)\n",
        "\n",
        "    def init_weights(self, m):\n",
        "        if isinstance(m, nn.Linear):\n",
        "            nn.init.xavier_normal_(m.weight)\n",
        "            nn.init.zeros_(m.bias)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.net(x)\n",
        "\n",
        "class GraphAttentionLayer(nn.Module):\n",
        "    def __init__(self, in_features, out_features, n_heads=3 , concat = True, dropout = 0.1, leaky_relu_slope = 0.1):\n",
        "        super(GraphAttentionLayer, self).__init__()\n",
        "\n",
        "        self.n_heads = n_heads # Number of attention heads\n",
        "        self.concat = concat # wether to concatenate the final attention heads\n",
        "        self.dropout = dropout # Dropout rate\n",
        "\n",
        "        # if concat: # concatenating the attention heads\n",
        "        #     self.out_features = out_features # Number of output features per node\n",
        "        #     assert out_features % n_heads == 0 # Ensure that out_features is a multiple of n_heads\n",
        "        #     self.n_hidden = out_features // n_heads\n",
        "        # else: # averaging output over the attention heads (Used in the main paper)\n",
        "        #     self.n_hidden = out_features\n",
        "        self.out_features = out_features\n",
        "        self.n_hidden = out_features\n",
        "\n",
        "        #  A shared linear transformation, parametrized by a weight matrix W is applied to every node\n",
        "        #  Initialize the weight matrix W\n",
        "        self.W = nn.Parameter(torch.empty(size=(in_features, self.n_hidden * n_heads)))\n",
        "\n",
        "        # Initialize the attention weights a\n",
        "        self.a = nn.Parameter(torch.empty(size=(n_heads, 2 * self.n_hidden, 1)))\n",
        "\n",
        "        self.leakyrelu = nn.LeakyReLU(leaky_relu_slope) # LeakyReLU activation function\n",
        "        self.softmax = nn.Softmax(dim=1) # softmax activation function to the attention coefficients\n",
        "\n",
        "        self.reset_parameters() # Reset the parameters\n",
        "\n",
        "\n",
        "    def reset_parameters(self):\n",
        "        \"\"\"\n",
        "        Reinitialize learnable parameters.\n",
        "        \"\"\"\n",
        "        nn.init.xavier_normal_(self.W)\n",
        "        nn.init.xavier_normal_(self.a)\n",
        "\n",
        "\n",
        "    def _get_attention_scores(self, h_transformed: torch.Tensor):\n",
        "\n",
        "        source_scores = torch.matmul(h_transformed, self.a[:, :self.n_hidden, :])\n",
        "        target_scores = torch.matmul(h_transformed, self.a[:, self.n_hidden:, :])\n",
        "\n",
        "        # broadcast add\n",
        "        # (n_heads, n_nodes, 1) + (n_heads, 1, n_nodes) = (n_heads, n_nodes, n_nodes)\n",
        "        e = source_scores + target_scores.mT\n",
        "        return self.leakyrelu(e)\n",
        "\n",
        "    def forward(self,  h: torch.Tensor, adj_mat: torch.Tensor):\n",
        "        n_nodes = h.shape[0]\n",
        "\n",
        "        # Apply linear transformation to node feature -> W h\n",
        "        # output shape (n_nodes, n_hidden * n_heads)\n",
        "        h_transformed = torch.mm(h, self.W)\n",
        "        h_transformed = F.dropout(h_transformed, self.dropout, training=self.training)\n",
        "\n",
        "        # splitting the heads by reshaping the tensor and putting heads dim first\n",
        "        # output shape (n_heads, n_nodes, n_hidden)\n",
        "        h_transformed = h_transformed.view(n_nodes, self.n_heads, self.n_hidden).permute(1, 0, 2)\n",
        "\n",
        "        # getting the attention scores\n",
        "        # output shape (n_heads, n_nodes, n_nodes)\n",
        "        e = self._get_attention_scores(h_transformed)\n",
        "\n",
        "        # Set the attention score for non-existent edges to -9e15 (MASKING NON-EXISTENT EDGES)\n",
        "        connectivity_mask = -9e16 * torch.ones_like(e)\n",
        "        adj_mat = adj_mat.to(device)\n",
        "        print(\"adj_matrix shape is -> \",np.shape(adj_mat))\n",
        "        e = torch.where(adj_mat > 0, e, connectivity_mask) # masked attention scores\n",
        "\n",
        "        # attention coefficients are computed as a softmax over the rows\n",
        "        # for each column j in the attention score matrix e\n",
        "        attention = F.softmax(e, dim=-1)\n",
        "        attention = F.dropout(attention, self.dropout, training=self.training)\n",
        "\n",
        "        # final node embeddings are computed as a weighted average of the features of its neighbors\n",
        "        h_prime = torch.matmul(attention, h_transformed)\n",
        "\n",
        "        # concatenating/averaging the attention heads\n",
        "        # output shape (n_nodes, out_features)\n",
        "        h_prime = h_prime.mean(dim=0)\n",
        "\n",
        "        return h_prime\n",
        "\n",
        "class MultiHeadGraphAttention(nn.Module):\n",
        "    \"\"\" multiple heads of graph self-attention in parallel \"\"\"\n",
        "    #documentation for GATConv class -> https://pytorch-geometric.readthedocs.io/en/latest/generated/torch_geometric.nn.conv.GATConv.html\n",
        "    def __init__(self, num_feats, h_channels=8, num_heads=3, dropout=0.1, alpha=0.2, masked=False):\n",
        "        super().__init__()\n",
        "        # self.conv = [GraphAttentionLayer(num_feats, h_channels, n_heads=num_heads, dropout=dropout, leaky_relu_slope=alpha) for _ in range(num_heads)]\n",
        "        # self.conv = GATConv(num_feats, h_channels, heads=num_heads, dropout=dropout, negative_slope=alpha)\n",
        "        self.conv = GraphAttentionLayer(num_feats, num_heads*h_channels, n_heads=num_heads, dropout=dropout, leaky_relu_slope=alpha)\n",
        "        self.proj = nn.Linear(h_channels*num_heads, num_feats)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "    # def __init__(self, in_features, out_features, n_heads=3 , concat = True, dropout = 0.1, leaky_relu_slope = 0.1):\n",
        "    def forward(self, x, edge_index, edge_attr):\n",
        "        # print(\"x is -> \",x, np.shape(x),type(x))\n",
        "        # print(\"edge idx is -> \",edge_index, np.shape(edge_index),type(edge_index))\n",
        "        adj_matrix = self.return_adj(edge_index)\n",
        "        # head_outputs = self.conv(x,edge_index,edge_attr) # List of head outputs\n",
        "        # head_outputs = torch.cat([att(x, adj_matrix) for att in self.conv], dim=1)\n",
        "        head_outputs = self.conv(x,adj_matrix)\n",
        "        print(\"head_outputs is \",head_outputs,np.shape(head_outputs),type(head_outputs))\n",
        "        out = self.dropout(head_outputs)\n",
        "        out = self.proj(out)\n",
        "        return out\n",
        "\n",
        "    def return_adj(self,edge_index):\n",
        "      global num_nodes\n",
        "      num_nodes = edge_index.max().item() + 1\n",
        "      # print(\"num_nodes local is -> \",num_nodes)\n",
        "      adjacency_matrix = torch.zeros((num_nodes, num_nodes), dtype=torch.float)\n",
        "      adjacency_matrix[edge_index[:, 0], edge_index[:, 1]] = 1\n",
        "      return adjacency_matrix\n",
        "\n",
        "class GraphBlock(nn.Module):\n",
        "    \"\"\" Transformer block: communication followed by computation \"\"\"\n",
        "    def __init__(self, num_feats, num_heads, h_channels, masked):\n",
        "        # num_feats: feature space dimension, n_head: the number of heads we'd like\n",
        "        super().__init__()\n",
        "        self.sa = MultiHeadGraphAttention(num_feats,num_heads=num_heads,h_channels=h_channels, masked=masked)\n",
        "        # print(\"number of feats in feedforward is -> \",num_feats)\n",
        "        self.ffwd = FeedForward(num_feats)\n",
        "        self.ln1 = nn.LayerNorm(num_feats)\n",
        "        self.ln2 = nn.LayerNorm(num_feats)\n",
        "\n",
        "    def forward(self, x, edge_index, edge_attr):\n",
        "        x = self.ln1(x)\n",
        "        # print(\"x_gb after layer_norm is -> \",x, np.shape(x),type(x))\n",
        "        x = self.sa(x,edge_index,edge_attr)\n",
        "        # print(\"x_gb after mhga is -> \",x, np.shape(x),type(x))\n",
        "        x = self.ln2(x)\n",
        "        # print(\"x_gb after layer_norm is -> \",x, np.shape(x),type(x))\n",
        "        x = x + self.ffwd(self.ln2(x))\n",
        "        return x\n",
        "\n",
        "class PriorDiscriminator(torch.nn.Module):\n",
        "    def __init__(self, input_dim):\n",
        "        super().__init__()\n",
        "        self.l0 = torch.nn.Linear(input_dim, input_dim)\n",
        "        self.l1 = torch.nn.Linear(input_dim, input_dim)\n",
        "        self.l2 = torch.nn.Linear(input_dim, 1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        h = F.relu(self.l0(x))\n",
        "        h = F.relu(self.l1(h))\n",
        "        return torch.sigmoid(self.l2(h))\n",
        "\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model: int, dropout: float = 0.2, max_len: int = 300):\n",
        "        super().__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        position = torch.arange(max_len).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n",
        "        pe = torch.zeros(max_len, 1, d_model)\n",
        "        # print(\"div_term is -> \",div_term, np.shape(div_term),type(div_term))\n",
        "        # print(\"pe is -> \",pe, np.shape(pe),type(pe))\n",
        "        # print(\"pe1 is -> \",pe[:, 0, 0::2],np.shape(pe[:, 0, 0::2]),type(pe[:, 0, 0::2]))\n",
        "        # print(\"pe2 is -> \",pe[:, 0, 1::3],np.shape(pe[:, 0, 1::3]),type(pe[:, 0, 1::3]))\n",
        "        pe[:, 0, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 0, 0::2] = torch.cos(position * div_term)\n",
        "        self.register_buffer('pe', pe)\n",
        "\n",
        "    def forward(self,x):\n",
        "        x = x + self.pe[:x.size(0)]\n",
        "        return self.dropout(x)\n",
        "\n",
        "class GraphTransformerEncoderModel(nn.Module):\n",
        "    def __init__(self, num_feats, num_embs, emb_dim, output_size, max_length, h_channels=8, num_heads=3, n_layers=2, decoder_bool=False, masked=False, train_embs = False):\n",
        "        super().__init__()\n",
        "        self.num_heads = num_heads\n",
        "        self.decoder_bool = decoder_bool\n",
        "        self.output_size = output_size\n",
        "        self.emb_layer = nn.Embedding(num_embs,emb_dim,sparse=True)\n",
        "        self.emb_layer.weight.requires_grad = train_embs\n",
        "        # self.pos_emb\n",
        "        if train_embs: self.init_weights(self.emb_layer)\n",
        "        self.pos_enc = PositionalEncoding(emb_dim,max_len=max_length)\n",
        "        self.blocks = nn.Sequential(*[GraphBlock(emb_dim*num_feats, num_heads, h_channels=h_channels, masked=masked) for _ in range(n_layers)])\n",
        "        self.ln_f = nn.LayerNorm(num_feats*emb_dim)  # final layer norm\n",
        "        self.output = nn.Linear(num_feats*emb_dim, output_size)\n",
        "        self.emb_act = nn.ReLU()\n",
        "\n",
        "    def forward(self, x, edge_index, edge_attr):\n",
        "        # x = self.emb_layer(x)\n",
        "        print(\"x before emb is -> \",x,np.shape(x))\n",
        "        x1 = self.emb_layer(x)\n",
        "        x2 = self.pos_enc(x1)\n",
        "        x = x1+x2\n",
        "        # print(\"x after emb is \",np.shape(x),np.shape(x1),np.shape(x2))\n",
        "        if(np.shape(x.shape)[0]==3):\n",
        "            B,C,D = x.shape\n",
        "            x = x.view(B, C*D)\n",
        "        print(\"x after emb. is -> \",np.shape(x))\n",
        "        for block in self.blocks: x = block(x, edge_index, edge_attr)\n",
        "        x = self.ln_f(x)\n",
        "        x = self.output(x)\n",
        "        if(self.decoder_bool==False):\n",
        "            x_relu = self.emb_act(x)\n",
        "            return x_relu\n",
        "        else: return x\n",
        "\n",
        "class FinalModel(nn.Module):\n",
        "    def __init__(self, num_feats, num_embs, emb_dim, output_size, max_length, processing_steps=3, beta=1.0):\n",
        "        super().__init__()\n",
        "        self.output_size = output_size\n",
        "        self.output_size_enc = output_size\n",
        "        self.output_size_disc = 1\n",
        "        self.emb_dim = emb_dim\n",
        "        self.num_nodes = num_nodes\n",
        "        self.beta = beta\n",
        "        self.encoder = GraphTransformerEncoderModel(num_feats,num_embs,emb_dim,self.output_size_enc,max_length)\n",
        "        self.aggr_oper = Set2Set(self.output_size_enc,processing_steps)\n",
        "        # gate_custom = nn.LSTM(self.output_size_enc,8)\n",
        "        # self.aggr_oper = AttentionalAggregationCustom(gate_custom)\n",
        "        # self.aggr_oper = MeanAggregation()\n",
        "        self.local_d = Discriminator(self.output_size_enc,self.output_size_disc)\n",
        "        self.global_d = Discriminator(2*self.output_size_enc,self.output_size_disc)\n",
        "        self.prior_d = PriorDiscriminator(self.output_size_disc)\n",
        "        self.mutual_l = MutualInformation()\n",
        "        self.cosine_l = nn.CosineSimilarity(dim=1, eps=1e-6)\n",
        "        self.prop_l = nn.BCELoss()\n",
        "        self.init_emb()\n",
        "        # self.cosine_l = nn.CosineEmbeddingLoss()\n",
        "\n",
        "    def forward(self, x, edge_index, edge_attr):\n",
        "        x_enc = self.encoder(x,edge_index,edge_attr)\n",
        "        B,C = x_enc.shape\n",
        "        print(\"shapes are -> \",B,\" \",C)\n",
        "        x_set = self.aggr_oper(x_enc,dim_size=1)\n",
        "        print(\"aggregator gives -> \",x_set,np.shape(x_set))\n",
        "        print(\"encoder gives -> \",x_enc,np.shape(x_enc))\n",
        "        x_enc_d = self.local_d(x_enc)\n",
        "        x_set_d = self.global_d(x_set)\n",
        "        # x_enc_d = torch.matmul(x_enc_d,x_set_d.t())\n",
        "        return x_enc_d, x_set_d\n",
        "\n",
        "    def mutual_loss(self, arr1, arr2):\n",
        "        return self.mutual_l(arr1,arr2)\n",
        "\n",
        "    def cosine_loss(self,arr1,arr2):\n",
        "        return self.cosine_l(arr1,arr2)\n",
        "\n",
        "    def probability_loss(self,arr1,arr2):\n",
        "        return self.prop_l(arr1,arr2)\n",
        "\n",
        "    def local_global_loss_(self,l_enc, g_enc, edge_index, batch):\n",
        "      num_nodes = l_enc.shape[0]\n",
        "      num_graphs = g_enc.shape[0]\n",
        "      print(\"num of nodes and num of graphs are -> \", num_nodes, \" \",num_graphs)\n",
        "\n",
        "      pos_mask = torch.zeros((num_nodes, num_graphs)).cuda()\n",
        "      neg_mask = torch.ones((num_nodes, num_graphs)).cuda()\n",
        "      for nodeidx, graphidx in enumerate(batch):\n",
        "          pos_mask[nodeidx][graphidx] = 1\n",
        "          neg_mask[nodeidx][graphidx] = 0\n",
        "\n",
        "      res = torch.mm(l_enc, g_enc.t())\n",
        "      log_2 = math.log(2.)\n",
        "\n",
        "      p_samples = res*pos_mask\n",
        "      E_pos = log_2 - F.softplus(- p_samples)\n",
        "      E_pos = E_pos.sum()\n",
        "      E_pos = E_pos/self.num_nodes\n",
        "\n",
        "      n_samples = res*neg_mask\n",
        "      E_neg = F.softplus(-n_samples) + n_samples - log_2\n",
        "      E_neg = E_neg.sum()\n",
        "      E_neg = E_neg / (num_nodes * (num_graphs - 1))\n",
        "\n",
        "      prior = torch.rand_like(g_enc)\n",
        "\n",
        "      term_a = torch.log(self.prior_d(prior)).mean()\n",
        "      term_b = torch.log(1.0 - self.prior_d(g_enc)).mean()\n",
        "      PRIOR = -(term_a + term_b) * self.beta\n",
        "\n",
        "      return (E_neg - E_pos) + PRIOR\n",
        "\n",
        "    def init_emb(self):\n",
        "      initrange = -1.5 / self.emb_dim\n",
        "      for m in self.modules():\n",
        "          if isinstance(m, nn.Linear):\n",
        "              torch.nn.init.xavier_uniform_(m.weight.data)\n",
        "              if m.bias is not None: m.bias.data.fill_(0.0)\n",
        "\n",
        "class Discriminator(torch.nn.Module):\n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super().__init__()\n",
        "        self.disc_block = torch.nn.Sequential(\n",
        "            torch.nn.Linear(input_dim, 2*input_dim),\n",
        "            torch.nn.ReLU(),\n",
        "            torch.nn.Linear(2*input_dim, 4*input_dim),\n",
        "            torch.nn.ReLU(),\n",
        "            torch.nn.Linear(4*input_dim, output_dim),\n",
        "            torch.nn.ReLU()\n",
        "        )\n",
        "        self.soft_ = torch.nn.Softmax()\n",
        "        self.sigm_ = nn.Sigmoid()\n",
        "        self.linear_shortcut = torch.nn.Linear(input_dim, output_dim)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x1 = self.disc_block(x)\n",
        "        x2 = self.linear_shortcut(x)\n",
        "        xol = x1+x2\n",
        "        return self.sigm_(xol)\n",
        "\n",
        "class MutualInformation(nn.Module):\n",
        "\tdef __init__(self, sigma=0.1, num_bins=256, normalize=True):\n",
        "\t\tsuper(MutualInformation, self).__init__()\n",
        "\t\tself.sigma = sigma\n",
        "\t\tself.num_bins = num_bins\n",
        "\t\tself.normalize = normalize\n",
        "\t\tself.epsilon = 1e-10\n",
        "\t\tself.bins = nn.Parameter(torch.linspace(0, 255, num_bins).float(), requires_grad=False)\n",
        "\n",
        "\tdef marginalPdf(self, values):\n",
        "\t\tresiduals = values - self.bins.unsqueeze(0).unsqueeze(0)\n",
        "\t\tkernel_values = torch.exp(-0.5*(residuals / self.sigma).pow(2))\n",
        "\t\tpdf = torch.mean(kernel_values, dim=1)\n",
        "\t\tnormalization = torch.sum(pdf, dim=1).unsqueeze(1) + self.epsilon\n",
        "\t\tpdf = pdf / normalization\n",
        "\t\treturn pdf, kernel_values\n",
        "\n",
        "\tdef jointPdf(self, kernel_values1, kernel_values2):\n",
        "\t\tjoint_kernel_values = torch.matmul(kernel_values1.transpose(1, 2), kernel_values2)\n",
        "\t\tnormalization = torch.sum(joint_kernel_values, dim=(1,2)).view(-1, 1, 1) + self.epsilon\n",
        "\t\tpdf = joint_kernel_values / normalization\n",
        "\t\treturn pdf\n",
        "\n",
        "\tdef getMutualInformation(self, input1, input2):\n",
        "\t\t# Torch tensors for images between (0, 1)\n",
        "\t\tinput1 = input1*256\n",
        "\t\tinput2 = input2*256\n",
        "\t\tB, C = input1.shape\n",
        "\t\tassert((input1.shape == input2.shape))\n",
        "\t\tx1 = input1.view(B, C, 1)\n",
        "\t\tx2 = input2.view(B, C, 1)\n",
        "\t\tpdf_x1, kernel_values1 = self.marginalPdf(x1)\n",
        "\t\tpdf_x2, kernel_values2 = self.marginalPdf(x2)\n",
        "\t\tpdf_x1x2 = self.jointPdf(kernel_values1, kernel_values2)\n",
        "\t\tH_x1 = -torch.sum(pdf_x1*torch.log2(pdf_x1 + self.epsilon), dim=1)\n",
        "\t\tH_x2 = -torch.sum(pdf_x2*torch.log2(pdf_x2 + self.epsilon), dim=1)\n",
        "\t\tH_x1x2 = -torch.sum(pdf_x1x2*torch.log2(pdf_x1x2 + self.epsilon), dim=(1,2))\n",
        "\t\tmutual_information = H_x1 + H_x2 - H_x1x2\n",
        "\t\tif self.normalize:\n",
        "\t\t\tmutual_information = 2*mutual_information/(H_x1+H_x2)\n",
        "\t\treturn mutual_information\n",
        "\n",
        "\tdef forward(self, input1, input2):\n",
        "\t\treturn self.getMutualInformation(input1, input2)\n",
        "\n",
        "print(\"loading of models done\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b6gngGewdMcy",
        "outputId": "e3cbf406-6cc6-4135-cc32-117bdd38e0e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "x bef is ->  tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x bef is ->  tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "data list is ->  [Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2]), Data(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2])] (20, 3, 2) <class 'list'>\n",
            "dataloader is ->  <torch_geometric.loader.dataloader.DataLoader object at 0x7c90b7ec8cd0> () <class 'torch_geometric.loader.dataloader.DataLoader'>\n",
            "\n",
            "pytorch version is:  2.0.1+cu118\n",
            "+------------------------------------+------------+\n",
            "|              Modules               | Parameters |\n",
            "+------------------------------------+------------+\n",
            "|     encoder.blocks.0.sa.conv.W     |   79200    |\n",
            "|     encoder.blocks.0.sa.conv.a     |    144     |\n",
            "|  encoder.blocks.0.sa.proj.weight   |   26400    |\n",
            "|   encoder.blocks.0.sa.proj.bias    |    1100    |\n",
            "| encoder.blocks.0.ffwd.net.0.weight |  4840000   |\n",
            "|  encoder.blocks.0.ffwd.net.0.bias  |    4400    |\n",
            "| encoder.blocks.0.ffwd.net.2.weight |  4840000   |\n",
            "|  encoder.blocks.0.ffwd.net.2.bias  |    1100    |\n",
            "|    encoder.blocks.0.ln1.weight     |    1100    |\n",
            "|     encoder.blocks.0.ln1.bias      |    1100    |\n",
            "|    encoder.blocks.0.ln2.weight     |    1100    |\n",
            "|     encoder.blocks.0.ln2.bias      |    1100    |\n",
            "|     encoder.blocks.1.sa.conv.W     |   79200    |\n",
            "|     encoder.blocks.1.sa.conv.a     |    144     |\n",
            "|  encoder.blocks.1.sa.proj.weight   |   26400    |\n",
            "|   encoder.blocks.1.sa.proj.bias    |    1100    |\n",
            "| encoder.blocks.1.ffwd.net.0.weight |  4840000   |\n",
            "|  encoder.blocks.1.ffwd.net.0.bias  |    4400    |\n",
            "| encoder.blocks.1.ffwd.net.2.weight |  4840000   |\n",
            "|  encoder.blocks.1.ffwd.net.2.bias  |    1100    |\n",
            "|    encoder.blocks.1.ln1.weight     |    1100    |\n",
            "|     encoder.blocks.1.ln1.bias      |    1100    |\n",
            "|    encoder.blocks.1.ln2.weight     |    1100    |\n",
            "|     encoder.blocks.1.ln2.bias      |    1100    |\n",
            "|        encoder.ln_f.weight         |    1100    |\n",
            "|         encoder.ln_f.bias          |    1100    |\n",
            "|       encoder.output.weight        |   12100    |\n",
            "|        encoder.output.bias         |     11     |\n",
            "|    aggr_oper.lstm.weight_ih_l0     |    968     |\n",
            "|    aggr_oper.lstm.weight_hh_l0     |    484     |\n",
            "|     aggr_oper.lstm.bias_ih_l0      |     44     |\n",
            "|     aggr_oper.lstm.bias_hh_l0      |     44     |\n",
            "|    local_d.disc_block.0.weight     |    242     |\n",
            "|     local_d.disc_block.0.bias      |     22     |\n",
            "|    local_d.disc_block.2.weight     |    968     |\n",
            "|     local_d.disc_block.2.bias      |     44     |\n",
            "|    local_d.disc_block.4.weight     |     44     |\n",
            "|     local_d.disc_block.4.bias      |     1      |\n",
            "|   local_d.linear_shortcut.weight   |     11     |\n",
            "|    local_d.linear_shortcut.bias    |     1      |\n",
            "|    global_d.disc_block.0.weight    |    968     |\n",
            "|     global_d.disc_block.0.bias     |     44     |\n",
            "|    global_d.disc_block.2.weight    |    3872    |\n",
            "|     global_d.disc_block.2.bias     |     88     |\n",
            "|    global_d.disc_block.4.weight    |     88     |\n",
            "|     global_d.disc_block.4.bias     |     1      |\n",
            "|  global_d.linear_shortcut.weight   |     22     |\n",
            "|   global_d.linear_shortcut.bias    |     1      |\n",
            "|         prior_d.l0.weight          |     1      |\n",
            "|          prior_d.l0.bias           |     1      |\n",
            "|         prior_d.l1.weight          |     1      |\n",
            "|          prior_d.l1.bias           |     1      |\n",
            "|         prior_d.l2.weight          |     1      |\n",
            "|          prior_d.l2.bias           |     1      |\n",
            "+------------------------------------+------------+\n",
            "Total Trainable Params: 19615762\n",
            "model gives ->  19615762\n",
            "device is ->  cuda True\n",
            "inside index i have 0 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 0 tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 1 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 1 tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 2 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 2 tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 3 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 3 tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 4 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 4 tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 5 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 5 tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 6 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 6 tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 7 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 7 tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 8 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 8 tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 9 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 9 tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 10 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 10 tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 11 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 11 tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 12 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 12 tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 13 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 13 tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 14 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 14 tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 15 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 15 tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 16 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 16 tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 17 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 17 tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 18 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 18 tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "inside index i have 19 DataBatch(x=[3000, 11], edge_index=[2, 6000], edge_attr=[6000, 2], batch=[3000], ptr=[2]) (5, 2) 1\n",
            "inside index_2 i have 19 tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]]) torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "dataloading finished\n"
          ]
        }
      ],
      "source": [
        "#data generation and loading\n",
        "for i in range(num_graphs): #create dummy data\n",
        "    edge_index = torch.randint(1,num_nodes,(2,num_edges))\n",
        "    x = torch.mul(torch.rand(num_nodes,num_node_feats),100).type(torch.int64)\n",
        "    print(\"x bef is -> \",x, np.shape(x),type(x))\n",
        "    edge_attr = torch.rand(num_edges,num_edge_feats)\n",
        "    data_temp = Data(x=x, edge_index=edge_index,edge_attr=edge_attr)\n",
        "    data_list.append(data_temp)\n",
        "print(\"data list is -> \",data_list,np.shape(data_list),type(data_list))\n",
        "dataloader = DataLoader(data_list, batch_size=batch_size, shuffle=True)\n",
        "print(\"dataloader is -> \",dataloader,np.shape(dataloader),type(dataloader))\n",
        "print()\n",
        "# DEVICE = 'cpu'\n",
        "model = FinalModel(num_feats=num_node_feats, num_embs=num_embs, emb_dim=emb_dim, output_size=num_node_feats, max_length=int(batch_size*num_nodes))\n",
        "torch.cuda.empty_cache()\n",
        "print(\"pytorch version is: \",torch.__version__)\n",
        "# set_max_split_size_mb(model, dummy_x, dummy_edge_index, dummy_edge_attr, max_split_size_mb)\n",
        "# print(summary(model,[(num_nodes,num_node_feats),(2,num_edges),(num_edges,num_edge_feats)]))\n",
        "print(\"model gives -> \",count_parameters(model))\n",
        "print(\"device is -> \",DEVICE, torch.cuda.is_available())\n",
        "model.to(DEVICE)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.0001) # Initialize the Adam optimizer.\n",
        "# optimizer = torch.optim.SGD(model.parameters(), lr=0.0001, momentum=0.9) #Initialize the SDG optimizer\n",
        "for index,batch in enumerate(dataloader):\n",
        "    print(\"inside index i have\",index,batch,np.shape(batch),batch.num_graphs)\n",
        "    print(\"inside index_2 i have\",index,batch.x,np.shape(batch.x),type(batch.x))\n",
        "print(\"dataloading finished\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n7vOqs8KmCtE",
        "outputId": "e8494ae3-e1b3-495c-8bae-703e780cfb4a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Sep 11 07:25:06 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.105.17   Driver Version: 525.105.17   CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   61C    P0    29W /  70W |   3865MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rU0hxtsCdm1S",
        "outputId": "45bcd403-8a68-4286-f5ec-7d296d2ff7f0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1;30;43mStreaming output truncated to the last 5000 lines.\u001b[0m\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3945,  0.1231, -0.0152, -0.2676, -0.0490,  0.0803, -0.0742,  0.1612,\n",
            "          0.1488,  0.3828, -0.0265,  0.0050,  1.9961,  0.1451,  1.9370,  0.0399,\n",
            "          0.0637,  1.0154,  1.3854,  0.0000,  1.9509,  0.1704]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7089, 0.0000,  ..., 0.0000, 1.4488, 0.2043],\n",
            "        [0.0000, 1.5528, 0.0000,  ..., 0.0000, 2.2413, 0.3035],\n",
            "        [0.0000, 1.8198, 0.0000,  ..., 0.0000, 1.7096, 0.2486],\n",
            "        ...,\n",
            "        [0.0000, 2.1291, 0.0000,  ..., 0.0000, 0.4621, 0.0893],\n",
            "        [0.0000, 0.8741, 0.0000,  ..., 0.0000, 1.9433, 0.0000],\n",
            "        [0.0000, 1.5633, 0.1545,  ..., 0.0000, 1.4337, 0.0393]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.6046],\n",
            "        [0.4476],\n",
            "        [0.5098],\n",
            "        ...,\n",
            "        [0.7609],\n",
            "        [0.3950],\n",
            "        [0.4720]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5861],\n",
            "        [0.5861],\n",
            "        [0.5861],\n",
            "        ...,\n",
            "        [0.5861],\n",
            "        [0.5861],\n",
            "        [0.5861]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4064, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4064, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4064], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 6\n",
            "node feats are tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0229,  0.0111,  0.0064,  ..., -0.2529, -0.1870, -0.0755],\n",
            "        [-0.0119,  0.0117,  0.0044,  ..., -0.2556, -0.1786, -0.0734],\n",
            "        [-0.0177,  0.0118,  0.0077,  ..., -0.2499, -0.1851, -0.0695],\n",
            "        ...,\n",
            "        [-0.0233,  0.0163,  0.0067,  ..., -0.2575, -0.1744, -0.0787],\n",
            "        [-0.0120,  0.0203,  0.0104,  ..., -0.2504, -0.1843, -0.0741],\n",
            "        [-0.0220,  0.0190,  0.0032,  ..., -0.2504, -0.1793, -0.0722]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2288,  0.1682, -0.6730,  ..., -0.8189,  0.1091, -0.6914],\n",
            "        [-1.2297,  0.1723, -0.6634,  ..., -0.8145,  0.1004, -0.6729],\n",
            "        [-1.2277,  0.1705, -0.6649,  ..., -0.8134,  0.1059, -0.6796],\n",
            "        ...,\n",
            "        [-1.2340,  0.1756, -0.6686,  ..., -0.8179,  0.1040, -0.6724],\n",
            "        [-1.2197,  0.1743, -0.6622,  ..., -0.8063,  0.1056, -0.6744],\n",
            "        [-1.2240,  0.1721, -0.6613,  ..., -0.8094,  0.1032, -0.6751]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3757,  0.1384, -0.0165, -0.2654, -0.0505,  0.0769, -0.0707,  0.1953,\n",
            "          0.1395,  0.3445,  0.0319,  0.0057,  1.7502,  0.2826,  1.8589,  0.0699,\n",
            "          0.0907,  1.0539,  1.3552,  0.0000,  1.9730,  0.1403]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1869, 0.0000,  ..., 0.0000, 2.0130, 0.3966],\n",
            "        [0.0000, 1.2827, 1.2510,  ..., 0.0000, 2.4701, 0.0000],\n",
            "        [0.0000, 1.8509, 0.2076,  ..., 0.0000, 1.9388, 0.0000],\n",
            "        ...,\n",
            "        [0.2533, 1.3670, 1.6138,  ..., 0.0000, 1.7136, 0.0000],\n",
            "        [0.0000, 1.4729, 0.2528,  ..., 0.0000, 1.5909, 0.0304],\n",
            "        [0.0000, 2.0394, 0.0153,  ..., 0.0000, 1.4982, 0.0926]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5236],\n",
            "        [0.2989],\n",
            "        [0.4589],\n",
            "        ...,\n",
            "        [0.3996],\n",
            "        [0.4957],\n",
            "        [0.5230]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.6004],\n",
            "        [0.6004],\n",
            "        [0.6004],\n",
            "        ...,\n",
            "        [0.6004],\n",
            "        [0.6004],\n",
            "        [0.6004]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4018, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4018, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4018], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 7\n",
            "node feats are tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0104,  0.0162,  0.0220,  ..., -0.2792, -0.1892, -0.0539],\n",
            "        [ 0.0066,  0.0054,  0.0234,  ..., -0.2755, -0.1947, -0.0604],\n",
            "        [ 0.0110,  0.0193,  0.0095,  ..., -0.2711, -0.1874, -0.0570],\n",
            "        ...,\n",
            "        [ 0.0047,  0.0159,  0.0221,  ..., -0.2732, -0.1938, -0.0589],\n",
            "        [-0.0014,  0.0098,  0.0093,  ..., -0.2706, -0.1938, -0.0490],\n",
            "        [ 0.0098,  0.0138,  0.0177,  ..., -0.2698, -0.1833, -0.0502]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4605,  0.2215, -0.8052,  ..., -0.9992,  0.0597, -0.6904],\n",
            "        [-1.4532,  0.2159, -0.8018,  ..., -0.9986,  0.0679, -0.6969],\n",
            "        [-1.4615,  0.2301, -0.8109,  ..., -1.0086,  0.0656, -0.6995],\n",
            "        ...,\n",
            "        [-1.4578,  0.2231, -0.8062,  ..., -1.0006,  0.0623, -0.6945],\n",
            "        [-1.4589,  0.2184, -0.8056,  ..., -1.0057,  0.0639, -0.7008],\n",
            "        [-1.4612,  0.2304, -0.8054,  ..., -1.0031,  0.0571, -0.6943]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4073,  0.1125, -0.0167, -0.2630, -0.0570,  0.0788, -0.0478,  0.1453,\n",
            "          0.1486,  0.3854,  0.0565,  0.0044,  1.9758,  0.1559,  1.8963,  0.0631,\n",
            "          0.0992,  1.0503,  1.2514,  0.0000,  1.8773,  0.3908]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.5574, 0.0000,  ..., 0.0000, 2.5111, 1.5291],\n",
            "        [0.0000, 2.0781, 0.0000,  ..., 0.0000, 1.7605, 0.3325],\n",
            "        [0.0000, 2.1101, 0.0000,  ..., 0.0000, 1.8467, 0.2288],\n",
            "        ...,\n",
            "        [0.0000, 1.4767, 0.0000,  ..., 0.0000, 1.0419, 0.0000],\n",
            "        [0.0000, 3.0580, 0.0000,  ..., 0.0000, 2.2034, 0.9210],\n",
            "        [0.0000, 2.0495, 0.0000,  ..., 0.0000, 1.7503, 0.4152]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.8736],\n",
            "        [0.5451],\n",
            "        [0.5391],\n",
            "        ...,\n",
            "        [0.5134],\n",
            "        [0.7917],\n",
            "        [0.5759]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5499],\n",
            "        [0.5499],\n",
            "        [0.5499],\n",
            "        ...,\n",
            "        [0.5499],\n",
            "        [0.5499],\n",
            "        [0.5499]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4113, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4113, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4113], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 8\n",
            "node feats are tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0324,  0.0044,  0.0121,  ..., -0.2537, -0.1597, -0.0454],\n",
            "        [ 0.0304,  0.0027,  0.0059,  ..., -0.2744, -0.1685, -0.0399],\n",
            "        [ 0.0291,  0.0063,  0.0204,  ..., -0.2576, -0.1558, -0.0484],\n",
            "        ...,\n",
            "        [ 0.0314,  0.0076,  0.0142,  ..., -0.2617, -0.1760, -0.0458],\n",
            "        [ 0.0233,  0.0065,  0.0059,  ..., -0.2633, -0.1678, -0.0450],\n",
            "        [ 0.0296,  0.0033,  0.0132,  ..., -0.2674, -0.1715, -0.0354]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3277,  0.2980, -0.7288,  ..., -0.9469, -0.0123, -0.6644],\n",
            "        [-1.3278,  0.3019, -0.7284,  ..., -0.9451, -0.0123, -0.6630],\n",
            "        [-1.3266,  0.2862, -0.7279,  ..., -0.9450, -0.0047, -0.6822],\n",
            "        ...,\n",
            "        [-1.3249,  0.3009, -0.7303,  ..., -0.9420, -0.0097, -0.6729],\n",
            "        [-1.3351,  0.2939, -0.7305,  ..., -0.9494, -0.0130, -0.6686],\n",
            "        [-1.3375,  0.3009, -0.7287,  ..., -0.9504, -0.0112, -0.6714]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3925,  0.1316, -0.0149, -0.2584, -0.0597,  0.0772, -0.0728,  0.1619,\n",
            "          0.1427,  0.3797,  0.0558,  0.0021,  1.8886,  0.1627,  1.9417,  0.1631,\n",
            "          0.0549,  1.1360,  1.4410,  0.0000,  2.0567,  0.3941]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.6609, 0.3801,  ..., 0.0000, 1.5026, 0.0669],\n",
            "        [0.0000, 1.8142, 0.0000,  ..., 0.0000, 1.8578, 0.4808],\n",
            "        [0.0000, 2.3556, 0.0067,  ..., 0.0000, 1.9611, 0.3948],\n",
            "        ...,\n",
            "        [0.0000, 2.1266, 0.0000,  ..., 0.0000, 2.7544, 0.0000],\n",
            "        [0.0000, 2.0299, 0.0040,  ..., 0.0000, 1.9578, 0.4757],\n",
            "        [0.0000, 2.2123, 0.0000,  ..., 0.0000, 1.9880, 0.1129]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4771],\n",
            "        [0.5627],\n",
            "        [0.6316],\n",
            "        ...,\n",
            "        [0.4348],\n",
            "        [0.5491],\n",
            "        [0.4497]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5575],\n",
            "        [0.5575],\n",
            "        [0.5575],\n",
            "        ...,\n",
            "        [0.5575],\n",
            "        [0.5575],\n",
            "        [0.5575]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4070], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 9\n",
            "node feats are tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 1.6332e-02,  1.0783e-02,  1.4153e-02,  ..., -2.5634e-01,\n",
            "         -1.6379e-01, -9.5226e-02],\n",
            "        [ 1.2043e-02,  1.1510e-02,  9.9546e-03,  ..., -2.5749e-01,\n",
            "         -1.5598e-01, -9.0107e-02],\n",
            "        [ 7.1418e-03,  3.3349e-03,  1.9061e-02,  ..., -2.6709e-01,\n",
            "         -1.5001e-01, -8.0211e-02],\n",
            "        ...,\n",
            "        [ 6.5357e-03,  1.3392e-02,  3.0095e-02,  ..., -2.6559e-01,\n",
            "         -1.5285e-01, -9.0280e-02],\n",
            "        [ 2.1166e-02,  2.6656e-04,  1.9475e-02,  ..., -2.5769e-01,\n",
            "         -1.6052e-01, -7.5728e-02],\n",
            "        [ 1.5678e-02,  5.7054e-03,  2.0958e-02,  ..., -2.6174e-01,\n",
            "         -1.4850e-01, -8.6192e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4038,  0.2652, -0.7485,  ..., -0.8947,  0.0520, -0.8537],\n",
            "        [-1.4099,  0.2613, -0.7499,  ..., -0.8980,  0.0529, -0.8559],\n",
            "        [-1.4192,  0.2634, -0.7566,  ..., -0.9080,  0.0574, -0.8643],\n",
            "        ...,\n",
            "        [-1.4158,  0.2717, -0.7551,  ..., -0.9033,  0.0579, -0.8548],\n",
            "        [-1.4249,  0.2689, -0.7537,  ..., -0.9038,  0.0484, -0.8581],\n",
            "        [-1.4129,  0.2571, -0.7508,  ..., -0.9038,  0.0597, -0.8632]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3999,  0.1276, -0.0149, -0.2662, -0.0512,  0.0813, -0.0688,  0.1646,\n",
            "          0.1482,  0.4033,  0.0196,  0.0060,  2.0095,  0.1966,  2.0876,  0.0830,\n",
            "          0.0507,  1.0680,  1.4236,  0.0000,  1.9708,  0.3370]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9145, 0.0000,  ..., 0.0000, 1.6667, 0.2014],\n",
            "        [0.0000, 2.1552, 0.0505,  ..., 0.0000, 1.8625, 0.3730],\n",
            "        [0.0000, 2.1182, 0.1047,  ..., 0.0000, 2.3347, 0.5105],\n",
            "        ...,\n",
            "        [0.0000, 1.7670, 0.0494,  ..., 0.0000, 1.5508, 0.1928],\n",
            "        [0.0000, 2.1532, 0.0000,  ..., 0.0000, 1.8874, 0.3326],\n",
            "        [0.0000, 1.6594, 0.0000,  ..., 0.0000, 2.2493, 0.5238]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5533],\n",
            "        [0.5662],\n",
            "        [0.6499],\n",
            "        ...,\n",
            "        [0.5001],\n",
            "        [0.5364],\n",
            "        [0.5361]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5712],\n",
            "        [0.5712],\n",
            "        [0.5712],\n",
            "        ...,\n",
            "        [0.5712],\n",
            "        [0.5712],\n",
            "        [0.5712]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4094, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4094, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4094], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 10\n",
            "node feats are tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0116, -0.0125,  0.0198,  ..., -0.2701, -0.1810, -0.0278],\n",
            "        [ 0.0097, -0.0138,  0.0161,  ..., -0.2823, -0.1756, -0.0329],\n",
            "        [ 0.0078, -0.0138,  0.0174,  ..., -0.2763, -0.1787, -0.0333],\n",
            "        ...,\n",
            "        [ 0.0108, -0.0181,  0.0069,  ..., -0.2825, -0.1803, -0.0308],\n",
            "        [ 0.0095, -0.0116,  0.0143,  ..., -0.2780, -0.1855, -0.0302],\n",
            "        [ 0.0118, -0.0139,  0.0077,  ..., -0.2720, -0.1725, -0.0301]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3425,  0.2849, -0.8294,  ..., -1.0136, -0.0089, -0.8757],\n",
            "        [-1.3589,  0.3034, -0.8347,  ..., -1.0203, -0.0191, -0.8660],\n",
            "        [-1.3444,  0.2912, -0.8295,  ..., -1.0142, -0.0179, -0.8646],\n",
            "        ...,\n",
            "        [-1.3474,  0.2842, -0.8292,  ..., -1.0156, -0.0130, -0.8743],\n",
            "        [-1.3587,  0.2978, -0.8358,  ..., -1.0254, -0.0160, -0.8753],\n",
            "        [-1.3528,  0.2946, -0.8265,  ..., -1.0202, -0.0186, -0.8675]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3796,  0.1260, -0.0138, -0.2604, -0.0500,  0.0718, -0.0853,  0.1615,\n",
            "          0.1382,  0.3755, -0.0645,  0.0028,  1.9659,  0.1091,  1.7930,  0.0633,\n",
            "          0.0271,  1.0594,  1.5252,  0.0000,  1.9743,  0.1241]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9766, 0.0000,  ..., 0.0000, 1.7806, 0.0000],\n",
            "        [0.0000, 1.7698, 0.0000,  ..., 0.0000, 2.0878, 0.3056],\n",
            "        [0.0000, 2.0534, 0.0000,  ..., 0.0000, 1.9076, 0.2540],\n",
            "        ...,\n",
            "        [0.0000, 2.2259, 0.0000,  ..., 0.0000, 1.8991, 0.0000],\n",
            "        [0.0000, 1.4977, 0.7515,  ..., 0.0000, 2.5311, 0.0000],\n",
            "        [0.0000, 2.1178, 0.0000,  ..., 0.0000, 1.9339, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4773],\n",
            "        [0.4999],\n",
            "        [0.5482],\n",
            "        ...,\n",
            "        [0.4232],\n",
            "        [0.2582],\n",
            "        [0.4475]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5887],\n",
            "        [0.5887],\n",
            "        [0.5887],\n",
            "        ...,\n",
            "        [0.5887],\n",
            "        [0.5887],\n",
            "        [0.5887]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4008, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4008, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4008], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 11\n",
            "node feats are tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0050,  0.0261,  0.0074,  ..., -0.2576, -0.1859, -0.0426],\n",
            "        [-0.0081,  0.0204,  0.0072,  ..., -0.2605, -0.1809, -0.0510],\n",
            "        [-0.0083,  0.0242,  0.0020,  ..., -0.2538, -0.1892, -0.0521],\n",
            "        ...,\n",
            "        [ 0.0013,  0.0201,  0.0058,  ..., -0.2639, -0.1801, -0.0512],\n",
            "        [-0.0133,  0.0273,  0.0052,  ..., -0.2574, -0.1813, -0.0539],\n",
            "        [-0.0114,  0.0248,  0.0014,  ..., -0.2644, -0.1866, -0.0490]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1630,  0.1865, -0.7229,  ..., -0.8707, -0.0441, -0.6439],\n",
            "        [-1.1732,  0.1943, -0.7321,  ..., -0.8797, -0.0490, -0.6435],\n",
            "        [-1.1762,  0.1875, -0.7326,  ..., -0.8821, -0.0450, -0.6514],\n",
            "        ...,\n",
            "        [-1.1651,  0.1785, -0.7239,  ..., -0.8740, -0.0441, -0.6448],\n",
            "        [-1.1703,  0.1895, -0.7259,  ..., -0.8721, -0.0438, -0.6412],\n",
            "        [-1.1734,  0.1838, -0.7290,  ..., -0.8771, -0.0436, -0.6461]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4032,  0.1226, -0.0177, -0.2676, -0.0548,  0.0839, -0.0445,  0.1744,\n",
            "          0.1487,  0.3735,  0.0968,  0.0025,  1.8713,  0.2743,  2.0102,  0.0795,\n",
            "          0.1275,  1.0502,  1.1930,  0.0000,  1.8867,  0.3463]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.0227, 0.1481,  ..., 0.0000, 1.8635, 0.2357],\n",
            "        [0.0000, 1.9942, 0.0128,  ..., 0.0000, 1.7848, 0.3076],\n",
            "        [0.0000, 2.0185, 0.1419,  ..., 0.0000, 1.8637, 0.2384],\n",
            "        ...,\n",
            "        [0.0000, 1.9308, 0.2071,  ..., 0.0000, 1.7926, 0.4281],\n",
            "        [0.0000, 1.9925, 0.2506,  ..., 0.0000, 1.8153, 0.3602],\n",
            "        [0.0000, 1.6887, 0.1170,  ..., 0.0000, 1.5251, 0.1157]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5461],\n",
            "        [0.5693],\n",
            "        [0.5458],\n",
            "        ...,\n",
            "        [0.5261],\n",
            "        [0.5788],\n",
            "        [0.5602]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5682],\n",
            "        [0.5682],\n",
            "        [0.5682],\n",
            "        ...,\n",
            "        [0.5682],\n",
            "        [0.5682],\n",
            "        [0.5682]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4118, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4118, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4118], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 12\n",
            "node feats are tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0156,  0.0109,  0.0214,  ..., -0.2680, -0.1768, -0.0492],\n",
            "        [-0.0112,  0.0118,  0.0197,  ..., -0.2690, -0.1859, -0.0475],\n",
            "        [-0.0156,  0.0200,  0.0259,  ..., -0.2721, -0.1858, -0.0475],\n",
            "        ...,\n",
            "        [-0.0117,  0.0144,  0.0092,  ..., -0.2680, -0.1750, -0.0369],\n",
            "        [-0.0188,  0.0127,  0.0148,  ..., -0.2707, -0.1897, -0.0448],\n",
            "        [-0.0077,  0.0118,  0.0172,  ..., -0.2721, -0.1869, -0.0479]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3206,  0.2300, -0.7062,  ..., -0.9676, -0.1042, -0.6803],\n",
            "        [-1.3268,  0.2425, -0.7149,  ..., -0.9676, -0.1070, -0.6827],\n",
            "        [-1.3356,  0.2456, -0.7140,  ..., -0.9738, -0.1073, -0.6743],\n",
            "        ...,\n",
            "        [-1.3236,  0.2424, -0.7099,  ..., -0.9660, -0.1068, -0.6669],\n",
            "        [-1.3366,  0.2410, -0.7139,  ..., -0.9752, -0.1052, -0.6870],\n",
            "        [-1.3340,  0.2394, -0.7118,  ..., -0.9729, -0.1028, -0.6807]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3759,  0.0985, -0.0182, -0.2510, -0.0514,  0.0717, -0.0309,  0.1764,\n",
            "          0.1349,  0.3813,  0.0878,  0.0113,  1.7614,  0.1945,  1.8593,  0.0880,\n",
            "          0.0667,  1.1492,  1.2840,  0.0000,  1.6163,  0.3896]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 0.9289, 0.0000,  ..., 0.0000, 1.6100, 0.4036],\n",
            "        [0.0000, 1.5687, 0.0000,  ..., 0.0000, 1.3614, 0.2592],\n",
            "        [0.0000, 1.1511, 0.0000,  ..., 0.0000, 1.7284, 0.6557],\n",
            "        ...,\n",
            "        [0.0000, 2.4798, 0.2514,  ..., 0.0000, 1.6123, 0.4213],\n",
            "        [0.0000, 1.6802, 0.0000,  ..., 0.0000, 1.2845, 0.3921],\n",
            "        [0.0000, 2.0262, 0.0000,  ..., 0.0000, 1.1774, 0.3016]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4652],\n",
            "        [0.4892],\n",
            "        [0.5688],\n",
            "        ...,\n",
            "        [0.6800],\n",
            "        [0.5720],\n",
            "        [0.5358]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5357],\n",
            "        [0.5357],\n",
            "        [0.5357],\n",
            "        ...,\n",
            "        [0.5357],\n",
            "        [0.5357],\n",
            "        [0.5357]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4112, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4112, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4112], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 13\n",
            "node feats are tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.8427e-02, -3.3727e-03,  9.9892e-03,  ..., -2.6642e-01,\n",
            "         -1.8453e-01, -6.0320e-02],\n",
            "        [-9.6183e-05, -1.0846e-02,  5.0201e-03,  ..., -2.7182e-01,\n",
            "         -1.6816e-01, -5.8401e-02],\n",
            "        [-1.5052e-02, -5.8595e-03, -8.7971e-04,  ..., -2.6706e-01,\n",
            "         -1.9022e-01, -6.1982e-02],\n",
            "        ...,\n",
            "        [-1.5568e-02, -5.0118e-03,  6.7361e-04,  ..., -2.7208e-01,\n",
            "         -1.8685e-01, -5.0746e-02],\n",
            "        [-1.2268e-02, -1.6126e-02,  2.3117e-03,  ..., -2.6000e-01,\n",
            "         -1.8109e-01, -5.4333e-02],\n",
            "        [-1.7620e-02, -1.2020e-02,  2.6697e-03,  ..., -2.6241e-01,\n",
            "         -1.9701e-01, -5.9695e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2049,  0.0490, -0.6240,  ..., -0.8955, -0.0307, -0.7241],\n",
            "        [-1.2011,  0.0506, -0.6244,  ..., -0.8923, -0.0278, -0.7295],\n",
            "        [-1.1922,  0.0666, -0.6243,  ..., -0.8906, -0.0327, -0.7208],\n",
            "        ...,\n",
            "        [-1.2045,  0.0410, -0.6242,  ..., -0.8887, -0.0255, -0.7310],\n",
            "        [-1.2061,  0.0580, -0.6290,  ..., -0.8905, -0.0300, -0.7227],\n",
            "        [-1.1959,  0.0549, -0.6223,  ..., -0.8910, -0.0255, -0.7227]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3800,  0.1093, -0.0181, -0.2581, -0.0500,  0.0764, -0.0572,  0.2009,\n",
            "          0.1434,  0.3668,  0.0732,  0.0037,  1.7474,  0.2163,  2.0113,  0.0914,\n",
            "          0.1080,  1.2307,  1.3254,  0.0000,  1.8637,  0.2394]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1012, 0.0000,  ..., 0.0000, 2.2063, 0.2971],\n",
            "        [0.0000, 1.7823, 0.0000,  ..., 0.0000, 2.1309, 0.0000],\n",
            "        [0.0000, 1.9130, 0.0000,  ..., 0.0000, 2.1714, 0.8135],\n",
            "        ...,\n",
            "        [0.0000, 1.1163, 0.0000,  ..., 0.0000, 1.9550, 0.1738],\n",
            "        [0.0000, 1.4382, 0.9363,  ..., 0.0000, 1.9491, 0.0000],\n",
            "        [0.0000, 1.6732, 0.0000,  ..., 0.0000, 1.7579, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.3960],\n",
            "        [0.4152],\n",
            "        [0.5631],\n",
            "        ...,\n",
            "        [0.5173],\n",
            "        [0.2840],\n",
            "        [0.4882]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5658],\n",
            "        [0.5658],\n",
            "        [0.5658],\n",
            "        ...,\n",
            "        [0.5658],\n",
            "        [0.5658],\n",
            "        [0.5658]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4052, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4052, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4052], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 14\n",
            "node feats are tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0158, -0.0217,  0.0280,  ..., -0.2985, -0.1631, -0.0504],\n",
            "        [-0.0183, -0.0130,  0.0413,  ..., -0.2891, -0.1646, -0.0549],\n",
            "        [-0.0200, -0.0212,  0.0358,  ..., -0.2850, -0.1570, -0.0579],\n",
            "        ...,\n",
            "        [-0.0113, -0.0142,  0.0317,  ..., -0.2884, -0.1556, -0.0517],\n",
            "        [-0.0230, -0.0220,  0.0231,  ..., -0.2791, -0.1614, -0.0508],\n",
            "        [-0.0122, -0.0191,  0.0323,  ..., -0.2864, -0.1576, -0.0559]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3468,  0.2129, -0.7191,  ..., -1.1438,  0.1500, -0.7970],\n",
            "        [-1.3532,  0.2141, -0.7187,  ..., -1.1502,  0.1429, -0.7953],\n",
            "        [-1.3634,  0.2151, -0.7226,  ..., -1.1578,  0.1439, -0.8032],\n",
            "        ...,\n",
            "        [-1.3677,  0.2195, -0.7218,  ..., -1.1593,  0.1450, -0.7973],\n",
            "        [-1.3527,  0.2148, -0.7175,  ..., -1.1518,  0.1405, -0.7915],\n",
            "        [-1.3532,  0.2099, -0.7160,  ..., -1.1535,  0.1384, -0.7955]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 3.7090e-01,  1.6600e-01, -1.4634e-02, -2.6349e-01, -5.4271e-02,\n",
            "          8.7458e-02, -9.4219e-02,  1.7642e-01,  1.5590e-01,  3.3239e-01,\n",
            "          4.0762e-02,  2.2263e-04,  1.6319e+00,  1.6423e-01,  1.9164e+00,\n",
            "          1.3853e-01,  6.5730e-02,  9.0149e-01,  1.3958e+00,  0.0000e+00,\n",
            "          2.1680e+00,  2.8126e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.3429, 0.2728,  ..., 0.0000, 2.2996, 0.0209],\n",
            "        [0.0000, 1.9346, 0.0000,  ..., 0.0000, 2.4498, 0.3858],\n",
            "        [0.0000, 2.9683, 0.0000,  ..., 0.0000, 2.1326, 1.0127],\n",
            "        ...,\n",
            "        [0.0000, 1.0942, 0.0000,  ..., 0.0000, 1.9249, 0.1956],\n",
            "        [0.0000, 1.7954, 0.0000,  ..., 0.0000, 2.3653, 0.4711],\n",
            "        [0.0000, 1.7048, 0.0000,  ..., 0.0000, 2.1296, 0.2250]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5504],\n",
            "        [0.4810],\n",
            "        [0.7382],\n",
            "        ...,\n",
            "        [0.4835],\n",
            "        [0.4781],\n",
            "        [0.4757]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5905],\n",
            "        [0.5905],\n",
            "        [0.5905],\n",
            "        ...,\n",
            "        [0.5905],\n",
            "        [0.5905],\n",
            "        [0.5905]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4040, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4040, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4040], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 15\n",
            "node feats are tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0004,  0.0440,  0.0029,  ..., -0.2664, -0.1833, -0.0635],\n",
            "        [-0.0163,  0.0521, -0.0052,  ..., -0.2628, -0.1804, -0.0565],\n",
            "        [-0.0012,  0.0547, -0.0008,  ..., -0.2758, -0.1767, -0.0538],\n",
            "        ...,\n",
            "        [ 0.0050,  0.0423, -0.0019,  ..., -0.2696, -0.1772, -0.0655],\n",
            "        [ 0.0017,  0.0501, -0.0013,  ..., -0.2698, -0.1765, -0.0650],\n",
            "        [-0.0084,  0.0485, -0.0030,  ..., -0.2676, -0.1881, -0.0598]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3221e+00,  1.4419e-01, -7.9696e-01,  ..., -8.8164e-01,\n",
            "         -1.5582e-03, -6.8008e-01],\n",
            "        [-1.3230e+00,  1.3815e-01, -7.9341e-01,  ..., -8.7571e-01,\n",
            "          3.2294e-03, -6.8465e-01],\n",
            "        [-1.3210e+00,  1.5068e-01, -7.9243e-01,  ..., -8.7775e-01,\n",
            "         -1.6508e-04, -6.8430e-01],\n",
            "        ...,\n",
            "        [-1.3211e+00,  1.4047e-01, -7.9149e-01,  ..., -8.8097e-01,\n",
            "          2.7644e-03, -6.8799e-01],\n",
            "        [-1.3267e+00,  1.4547e-01, -7.9840e-01,  ..., -8.8204e-01,\n",
            "          5.2375e-03, -6.8771e-01],\n",
            "        [-1.3190e+00,  1.3932e-01, -7.9089e-01,  ..., -8.7682e-01,\n",
            "          5.7671e-03, -6.8975e-01]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3969,  0.1045, -0.0189, -0.2610, -0.0576,  0.0721, -0.0302,  0.1614,\n",
            "          0.1371,  0.3568,  0.0807,  0.0124,  1.8648,  0.2499,  1.7209,  0.0482,\n",
            "          0.1210,  1.0924,  1.1516,  0.0000,  1.7626,  0.2986]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9929, 0.0613,  ..., 0.0000, 1.7167, 0.1980],\n",
            "        [0.0000, 1.8514, 0.0000,  ..., 0.0000, 1.6840, 0.0786],\n",
            "        [0.0000, 1.2947, 0.2761,  ..., 0.0000, 0.8587, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.1059, 0.1657,  ..., 0.0000, 1.6829, 0.0627],\n",
            "        [0.0000, 1.5838, 0.4474,  ..., 0.0000, 1.2412, 0.0000],\n",
            "        [0.0000, 2.5849, 0.4234,  ..., 0.0000, 1.8259, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5205],\n",
            "        [0.5166],\n",
            "        [0.5064],\n",
            "        ...,\n",
            "        [0.4864],\n",
            "        [0.4894],\n",
            "        [0.5516]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5584],\n",
            "        [0.5584],\n",
            "        [0.5584],\n",
            "        ...,\n",
            "        [0.5584],\n",
            "        [0.5584],\n",
            "        [0.5584]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4095, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4095, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4095], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 16\n",
            "node feats are tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0202,  0.0160,  0.0280,  ..., -0.2808, -0.1773, -0.0578],\n",
            "        [-0.0166,  0.0086,  0.0158,  ..., -0.2789, -0.1747, -0.0674],\n",
            "        [-0.0201,  0.0132,  0.0278,  ..., -0.2776, -0.1729, -0.0630],\n",
            "        ...,\n",
            "        [-0.0044,  0.0163,  0.0254,  ..., -0.2879, -0.1764, -0.0486],\n",
            "        [-0.0191, -0.0003,  0.0189,  ..., -0.2773, -0.1867, -0.0648],\n",
            "        [-0.0179,  0.0059,  0.0251,  ..., -0.2733, -0.1852, -0.0459]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2343,  0.2066, -0.7247,  ..., -0.9112, -0.1239, -0.8339],\n",
            "        [-1.2416,  0.1967, -0.7262,  ..., -0.9114, -0.1243, -0.8463],\n",
            "        [-1.2321,  0.2068, -0.7222,  ..., -0.9069, -0.1326, -0.8273],\n",
            "        ...,\n",
            "        [-1.2273,  0.1852, -0.7216,  ..., -0.9056, -0.1238, -0.8403],\n",
            "        [-1.2419,  0.2071, -0.7292,  ..., -0.9196, -0.1283, -0.8401],\n",
            "        [-1.2309,  0.2066, -0.7222,  ..., -0.9096, -0.1341, -0.8264]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3748,  0.1169, -0.0161, -0.2607, -0.0441,  0.0804, -0.0618,  0.1838,\n",
            "          0.1421,  0.3840,  0.0128,  0.0061,  1.8517,  0.1805,  1.9883,  0.0743,\n",
            "          0.0488,  1.0445,  1.3796,  0.0000,  1.7563,  0.1920]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9917, 0.0000,  ..., 0.0000, 1.7323, 0.0144],\n",
            "        [0.0000, 2.6946, 0.4452,  ..., 0.0000, 2.5548, 0.7097],\n",
            "        [0.0000, 1.8021, 0.0000,  ..., 0.0000, 1.6429, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.2783, 0.0000,  ..., 0.0000, 1.9300, 0.4888],\n",
            "        [0.0000, 0.7158, 0.0000,  ..., 0.0000, 1.4343, 0.0000],\n",
            "        [0.0000, 2.1147, 0.1346,  ..., 0.0000, 1.6953, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4821],\n",
            "        [0.6375],\n",
            "        [0.5101],\n",
            "        ...,\n",
            "        [0.5435],\n",
            "        [0.4679],\n",
            "        [0.4615]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5769],\n",
            "        [0.5769],\n",
            "        [0.5769],\n",
            "        ...,\n",
            "        [0.5769],\n",
            "        [0.5769],\n",
            "        [0.5769]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4054, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4054, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4054], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 17\n",
            "node feats are tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-8.7505e-03,  1.6105e-02,  3.6267e-03,  ..., -2.6579e-01,\n",
            "         -1.6797e-01, -5.8304e-02],\n",
            "        [ 4.4275e-04,  1.5369e-02,  5.6993e-03,  ..., -2.7479e-01,\n",
            "         -1.6792e-01, -6.9091e-02],\n",
            "        [-1.0210e-02,  1.4300e-02,  8.8973e-03,  ..., -2.7299e-01,\n",
            "         -1.6979e-01, -5.7543e-02],\n",
            "        ...,\n",
            "        [-2.9851e-03,  2.2604e-02,  1.0198e-03,  ..., -2.7398e-01,\n",
            "         -1.6839e-01, -5.2802e-02],\n",
            "        [-2.7587e-03,  1.6106e-02,  5.8192e-03,  ..., -2.7446e-01,\n",
            "         -1.7289e-01, -7.5305e-02],\n",
            "        [-7.1692e-03,  1.7217e-02, -2.0887e-04,  ..., -2.7053e-01,\n",
            "         -1.6936e-01, -6.8490e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4216,  0.0518, -0.7669,  ..., -1.0416,  0.1426, -0.7872],\n",
            "        [-1.4288,  0.0631, -0.7759,  ..., -1.0470,  0.1364, -0.7827],\n",
            "        [-1.4357,  0.0631, -0.7796,  ..., -1.0532,  0.1377, -0.7837],\n",
            "        ...,\n",
            "        [-1.4435,  0.0485, -0.7770,  ..., -1.0579,  0.1433, -0.8008],\n",
            "        [-1.4241,  0.0588, -0.7712,  ..., -1.0468,  0.1369, -0.7814],\n",
            "        [-1.4254,  0.0643, -0.7696,  ..., -1.0465,  0.1299, -0.7778]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4069,  0.1113, -0.0188, -0.2675, -0.0548,  0.0834, -0.0477,  0.1665,\n",
            "          0.1577,  0.3547,  0.0830,  0.0071,  1.8453,  0.1898,  1.9294,  0.0319,\n",
            "          0.1721,  1.0644,  1.1186,  0.0000,  1.9212,  0.2756]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.2836, 0.3128,  ..., 0.0000, 1.5578, 0.2946],\n",
            "        [0.0000, 1.4279, 0.0000,  ..., 0.0000, 1.8789, 0.0000],\n",
            "        [0.0000, 1.4974, 0.2201,  ..., 0.0000, 1.3857, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.9376, 0.0000,  ..., 0.0000, 1.9190, 0.1879],\n",
            "        [0.0000, 1.9555, 0.0000,  ..., 0.0000, 1.8796, 0.2216],\n",
            "        [0.0000, 2.1637, 0.0000,  ..., 0.0000, 1.5370, 0.0971]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5406],\n",
            "        [0.4865],\n",
            "        [0.4613],\n",
            "        ...,\n",
            "        [0.5283],\n",
            "        [0.5403],\n",
            "        [0.5041]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5654],\n",
            "        [0.5654],\n",
            "        [0.5654],\n",
            "        ...,\n",
            "        [0.5654],\n",
            "        [0.5654],\n",
            "        [0.5654]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4109, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4109, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4109], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 18\n",
            "node feats are tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0045, -0.0140,  0.0255,  ..., -0.2666, -0.1622, -0.0424],\n",
            "        [ 0.0007, -0.0134,  0.0169,  ..., -0.2719, -0.1733, -0.0390],\n",
            "        [-0.0178, -0.0067,  0.0224,  ..., -0.2693, -0.1601, -0.0292],\n",
            "        ...,\n",
            "        [ 0.0108, -0.0034,  0.0123,  ..., -0.2675, -0.1514, -0.0310],\n",
            "        [-0.0005, -0.0022,  0.0237,  ..., -0.2711, -0.1617, -0.0334],\n",
            "        [ 0.0017, -0.0089,  0.0264,  ..., -0.2708, -0.1579, -0.0300]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3936,  0.1996, -0.8830,  ..., -1.1675,  0.0856, -0.9540],\n",
            "        [-1.4019,  0.2140, -0.8894,  ..., -1.1721,  0.0813, -0.9461],\n",
            "        [-1.3851,  0.2016, -0.8787,  ..., -1.1561,  0.0842, -0.9420],\n",
            "        ...,\n",
            "        [-1.3999,  0.2069, -0.8875,  ..., -1.1704,  0.0861, -0.9523],\n",
            "        [-1.4002,  0.2068, -0.8855,  ..., -1.1715,  0.0850, -0.9521],\n",
            "        [-1.3967,  0.2004, -0.8810,  ..., -1.1718,  0.0834, -0.9516]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3662,  0.1548, -0.0122, -0.2541, -0.0480,  0.0757, -0.0952,  0.1758,\n",
            "          0.1418,  0.3859, -0.0343,  0.0026,  1.8242,  0.1297,  1.9541,  0.0899,\n",
            "          0.0350,  1.0032,  1.6733,  0.0000,  2.0256,  0.2872]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.5917, 0.1064,  ..., 0.0000, 2.1054, 0.0348],\n",
            "        [0.0000, 1.4268, 0.2147,  ..., 0.0000, 1.3534, 0.0000],\n",
            "        [0.0000, 2.4229, 0.0000,  ..., 0.0000, 2.0732, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.5895, 0.0000,  ..., 0.0000, 1.7044, 0.1054],\n",
            "        [0.0000, 1.5344, 0.0000,  ..., 0.0000, 2.2219, 0.4698],\n",
            "        [0.0000, 1.9859, 0.0000,  ..., 0.0000, 1.8804, 0.3743]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5489],\n",
            "        [0.4025],\n",
            "        [0.5828],\n",
            "        ...,\n",
            "        [0.3977],\n",
            "        [0.4779],\n",
            "        [0.5440]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5848],\n",
            "        [0.5848],\n",
            "        [0.5848],\n",
            "        ...,\n",
            "        [0.5848],\n",
            "        [0.5848],\n",
            "        [0.5848]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4023, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4023, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4023], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 19\n",
            "node feats are tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0103, -0.0236,  0.0097,  ..., -0.2626, -0.1872, -0.0453],\n",
            "        [ 0.0097, -0.0178,  0.0100,  ..., -0.2660, -0.1852, -0.0489],\n",
            "        [-0.0086, -0.0218,  0.0047,  ..., -0.2810, -0.1944, -0.0565],\n",
            "        ...,\n",
            "        [-0.0068, -0.0107,  0.0076,  ..., -0.2796, -0.1793, -0.0594],\n",
            "        [-0.0004, -0.0158,  0.0018,  ..., -0.2675, -0.1872, -0.0504],\n",
            "        [-0.0060, -0.0239,  0.0050,  ..., -0.2852, -0.1846, -0.0463]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4532,  0.1644, -0.7673,  ..., -1.0503,  0.0687, -0.7733],\n",
            "        [-1.4617,  0.1812, -0.7746,  ..., -1.0604,  0.0601, -0.7626],\n",
            "        [-1.4582,  0.1564, -0.7690,  ..., -1.0576,  0.0699, -0.7803],\n",
            "        ...,\n",
            "        [-1.4689,  0.1667, -0.7743,  ..., -1.0622,  0.0634, -0.7778],\n",
            "        [-1.4611,  0.1571, -0.7696,  ..., -1.0595,  0.0721, -0.7814],\n",
            "        [-1.4561,  0.1564, -0.7673,  ..., -1.0520,  0.0686, -0.7726]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3878,  0.1052, -0.0166, -0.2551, -0.0490,  0.0777, -0.0585,  0.1794,\n",
            "          0.1493,  0.3984,  0.0527,  0.0049,  1.8606,  0.1223,  2.0919,  0.0753,\n",
            "          0.0942,  1.1857,  1.3931,  0.0000,  1.8297,  0.3566]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.3046, 0.0302,  ..., 0.0000, 1.4822, 0.3656],\n",
            "        [0.0000, 2.8958, 0.0000,  ..., 0.0000, 1.9228, 0.7157],\n",
            "        [0.0000, 1.9743, 0.0000,  ..., 0.0000, 1.7016, 0.3996],\n",
            "        ...,\n",
            "        [0.0000, 3.6229, 0.0000,  ..., 0.0000, 1.9560, 1.1282],\n",
            "        [0.0000, 1.9673, 0.0000,  ..., 0.0000, 1.7768, 0.2800],\n",
            "        [0.0000, 1.3116, 0.0545,  ..., 0.0000, 1.4590, 0.3946]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5120],\n",
            "        [0.7005],\n",
            "        [0.5666],\n",
            "        ...,\n",
            "        [0.8735],\n",
            "        [0.5131],\n",
            "        [0.5206]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5471],\n",
            "        [0.5471],\n",
            "        [0.5471],\n",
            "        ...,\n",
            "        [0.5471],\n",
            "        [0.5471],\n",
            "        [0.5471]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4100, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4100, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4100], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "===== Epoch 3, Loss tensor([28.1284], grad_fn=<DivBackward0>) =====\n",
            "index of dataloader is 0\n",
            "node feats are tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0005,  0.0373, -0.0215,  ..., -0.2412, -0.1959, -0.0711],\n",
            "        [ 0.0037,  0.0291, -0.0284,  ..., -0.2646, -0.1890, -0.0674],\n",
            "        [ 0.0067,  0.0351, -0.0210,  ..., -0.2637, -0.1878, -0.0629],\n",
            "        ...,\n",
            "        [ 0.0141,  0.0294, -0.0264,  ..., -0.2664, -0.1937, -0.0751],\n",
            "        [ 0.0046,  0.0270, -0.0227,  ..., -0.2509, -0.1818, -0.0735],\n",
            "        [ 0.0029,  0.0268, -0.0267,  ..., -0.2524, -0.1928, -0.0662]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2796, -0.0697, -0.8249,  ..., -0.9398,  0.0827, -0.6464],\n",
            "        [-1.2873, -0.0551, -0.8318,  ..., -0.9439,  0.0830, -0.6468],\n",
            "        [-1.2793, -0.0585, -0.8311,  ..., -0.9436,  0.0810, -0.6432],\n",
            "        ...,\n",
            "        [-1.2776, -0.0637, -0.8285,  ..., -0.9411,  0.0794, -0.6469],\n",
            "        [-1.2864, -0.0568, -0.8342,  ..., -0.9460,  0.0754, -0.6412],\n",
            "        [-1.2797, -0.0570, -0.8251,  ..., -0.9393,  0.0767, -0.6455]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3973,  0.1003, -0.0221, -0.2616, -0.0556,  0.0846, -0.0486,  0.1921,\n",
            "          0.1643,  0.2956,  0.1404,  0.0037,  1.5959,  0.1531,  1.8824,  0.0745,\n",
            "          0.2604,  1.2025,  0.9836,  0.0000,  1.9934,  0.1931]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.2563, 0.4747,  ..., 0.0000, 2.9211, 0.7857],\n",
            "        [0.0000, 1.3154, 0.2798,  ..., 0.0000, 1.5516, 0.0000],\n",
            "        [0.0000, 1.1178, 0.0379,  ..., 0.0000, 1.6583, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 3.0625, 0.0000,  ..., 0.0000, 2.6166, 0.4410],\n",
            "        [0.0000, 1.9194, 0.0000,  ..., 0.0000, 1.6119, 0.0000],\n",
            "        [0.0000, 1.7418, 0.0000,  ..., 0.0000, 1.4103, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.7160],\n",
            "        [0.4863],\n",
            "        [0.4514],\n",
            "        ...,\n",
            "        [0.8290],\n",
            "        [0.4593],\n",
            "        [0.4717]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5594],\n",
            "        [0.5594],\n",
            "        [0.5594],\n",
            "        ...,\n",
            "        [0.5594],\n",
            "        [0.5594],\n",
            "        [0.5594]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4079, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4079, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4079], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 1\n",
            "node feats are tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0022, -0.0063,  0.0094,  ..., -0.2846, -0.1737, -0.0425],\n",
            "        [-0.0038, -0.0015,  0.0076,  ..., -0.2928, -0.1665, -0.0494],\n",
            "        [-0.0052, -0.0071,  0.0178,  ..., -0.2792, -0.1660, -0.0395],\n",
            "        ...,\n",
            "        [-0.0053, -0.0075,  0.0050,  ..., -0.2868, -0.1711, -0.0438],\n",
            "        [-0.0055, -0.0124,  0.0121,  ..., -0.2768, -0.1635, -0.0417],\n",
            "        [-0.0063, -0.0132,  0.0156,  ..., -0.2881, -0.1644, -0.0400]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4169,  0.1230, -0.7433,  ..., -1.1166,  0.0821, -0.8283],\n",
            "        [-1.4190,  0.1189, -0.7473,  ..., -1.1149,  0.0898, -0.8385],\n",
            "        [-1.4232,  0.1256, -0.7504,  ..., -1.1233,  0.0841, -0.8334],\n",
            "        ...,\n",
            "        [-1.4137,  0.1205, -0.7426,  ..., -1.1061,  0.0858, -0.8253],\n",
            "        [-1.4238,  0.1418, -0.7493,  ..., -1.1169,  0.0826, -0.8193],\n",
            "        [-1.4249,  0.1197, -0.7490,  ..., -1.1227,  0.0869, -0.8364]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3732,  0.1107, -0.0162, -0.2512, -0.0485,  0.0710, -0.0646,  0.1855,\n",
            "          0.1412,  0.3828,  0.0244,  0.0056,  1.7862,  0.1307,  1.9385,  0.0535,\n",
            "          0.0881,  1.1891,  1.4756,  0.0000,  1.8311,  0.2862]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.6696, 0.0000,  ..., 0.0000, 1.5806, 0.1442],\n",
            "        [0.0000, 1.1244, 0.0110,  ..., 0.0000, 0.8717, 0.0587],\n",
            "        [0.0000, 1.8916, 0.0000,  ..., 0.0000, 1.7888, 0.2145],\n",
            "        ...,\n",
            "        [0.0000, 1.0037, 0.0000,  ..., 0.0000, 1.8339, 0.5628],\n",
            "        [0.0000, 1.8886, 0.0000,  ..., 0.0000, 1.7095, 0.2607],\n",
            "        [0.0000, 1.6787, 0.0000,  ..., 0.0000, 1.4825, 0.3008]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4937],\n",
            "        [0.4913],\n",
            "        [0.4746],\n",
            "        ...,\n",
            "        [0.5103],\n",
            "        [0.4667],\n",
            "        [0.5435]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5581],\n",
            "        [0.5581],\n",
            "        [0.5581],\n",
            "        ...,\n",
            "        [0.5581],\n",
            "        [0.5581],\n",
            "        [0.5581]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4060, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4060, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4060], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 2\n",
            "node feats are tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.9842e-02,  6.4356e-03,  4.9783e-03,  ..., -2.7746e-01,\n",
            "         -1.9219e-01, -4.2699e-02],\n",
            "        [-2.4443e-02, -8.9114e-03,  1.2619e-02,  ..., -2.7663e-01,\n",
            "         -1.8205e-01, -3.6948e-02],\n",
            "        [-1.5764e-02, -2.4765e-03,  1.2188e-02,  ..., -2.8325e-01,\n",
            "         -1.8497e-01, -3.9156e-02],\n",
            "        ...,\n",
            "        [-2.2289e-02,  1.8619e-03,  1.8055e-02,  ..., -2.8537e-01,\n",
            "         -1.8997e-01, -3.7870e-02],\n",
            "        [-1.6149e-02, -4.9278e-06,  1.3435e-02,  ..., -2.7603e-01,\n",
            "         -1.8455e-01, -5.1738e-02],\n",
            "        [-2.2727e-02, -5.8418e-03,  1.8782e-02,  ..., -2.7176e-01,\n",
            "         -1.9179e-01, -4.2477e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3762,  0.2141, -0.7246,  ..., -1.0531, -0.0763, -0.7678],\n",
            "        [-1.3809,  0.2156, -0.7268,  ..., -1.0539, -0.0760, -0.7781],\n",
            "        [-1.3655,  0.2111, -0.7194,  ..., -1.0468, -0.0745, -0.7665],\n",
            "        ...,\n",
            "        [-1.3763,  0.2201, -0.7294,  ..., -1.0522, -0.0772, -0.7751],\n",
            "        [-1.3727,  0.2142, -0.7236,  ..., -1.0501, -0.0758, -0.7675],\n",
            "        [-1.3795,  0.2172, -0.7248,  ..., -1.0518, -0.0785, -0.7677]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3647,  0.1056, -0.0179, -0.2530, -0.0477,  0.0724, -0.0461,  0.1824,\n",
            "          0.1380,  0.3652,  0.0535,  0.0098,  1.6922,  0.1595,  1.8274,  0.0728,\n",
            "          0.0509,  1.1081,  1.3122,  0.0000,  1.6687,  0.2833]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.3646, 0.2401,  ..., 0.0000, 1.1320, 0.0000],\n",
            "        [0.0000, 1.6383, 0.0000,  ..., 0.0000, 1.5073, 0.0916],\n",
            "        [0.0000, 1.8393, 0.0000,  ..., 0.0000, 1.9215, 0.5815],\n",
            "        ...,\n",
            "        [0.0000, 2.3034, 0.0000,  ..., 0.0000, 1.2505, 0.6137],\n",
            "        [0.0000, 2.6946, 0.0000,  ..., 0.0000, 1.1568, 1.0424],\n",
            "        [0.0000, 2.6792, 0.2114,  ..., 0.0000, 2.3014, 0.6348]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4265],\n",
            "        [0.5153],\n",
            "        [0.5319],\n",
            "        ...,\n",
            "        [0.6988],\n",
            "        [0.8124],\n",
            "        [0.4954]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5493],\n",
            "        [0.5493],\n",
            "        [0.5493],\n",
            "        ...,\n",
            "        [0.5493],\n",
            "        [0.5493],\n",
            "        [0.5493]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4073, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4073, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4073], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 3\n",
            "node feats are tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0024,  0.0014, -0.0044,  ..., -0.2714, -0.1738, -0.0759],\n",
            "        [ 0.0029,  0.0112, -0.0135,  ..., -0.2728, -0.1786, -0.0703],\n",
            "        [-0.0046, -0.0022, -0.0041,  ..., -0.2715, -0.1855, -0.0786],\n",
            "        ...,\n",
            "        [ 0.0018,  0.0032, -0.0035,  ..., -0.2707, -0.1776, -0.0671],\n",
            "        [-0.0102, -0.0024, -0.0088,  ..., -0.2644, -0.1851, -0.0682],\n",
            "        [-0.0110,  0.0053, -0.0039,  ..., -0.2680, -0.1792, -0.0663]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2033,  0.0820, -0.6129,  ..., -0.7668, -0.0116, -0.8010],\n",
            "        [-1.2000,  0.0777, -0.6086,  ..., -0.7633, -0.0088, -0.8031],\n",
            "        [-1.2101,  0.0794, -0.6141,  ..., -0.7694, -0.0149, -0.8045],\n",
            "        ...,\n",
            "        [-1.2035,  0.0843, -0.6144,  ..., -0.7680, -0.0160, -0.8080],\n",
            "        [-1.2035,  0.0821, -0.6116,  ..., -0.7647, -0.0158, -0.8026],\n",
            "        [-1.2025,  0.0799, -0.6105,  ..., -0.7652, -0.0128, -0.8025]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3881,  0.1070, -0.0170, -0.2619, -0.0486,  0.0766, -0.0604,  0.1920,\n",
            "          0.1422,  0.3875,  0.0360,  0.0061,  1.8926,  0.2201,  2.0527,  0.0721,\n",
            "          0.0780,  1.2190,  1.3734,  0.0000,  1.8625,  0.2095]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7942, 0.0000,  ..., 0.0000, 1.7366, 0.0000],\n",
            "        [0.0000, 2.5332, 0.0000,  ..., 0.0000, 1.9755, 0.0000],\n",
            "        [0.0000, 2.1668, 0.0000,  ..., 0.0000, 2.2906, 0.4052],\n",
            "        ...,\n",
            "        [0.0000, 2.0373, 0.0334,  ..., 0.0000, 1.8127, 0.1192],\n",
            "        [0.0000, 1.2984, 0.0000,  ..., 0.0000, 1.9627, 0.4514],\n",
            "        [0.0000, 2.4754, 0.2898,  ..., 0.0000, 1.9025, 0.0360]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5012],\n",
            "        [0.4137],\n",
            "        [0.4969],\n",
            "        ...,\n",
            "        [0.4979],\n",
            "        [0.5265],\n",
            "        [0.5683]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5730],\n",
            "        [0.5730],\n",
            "        [0.5730],\n",
            "        ...,\n",
            "        [0.5730],\n",
            "        [0.5730],\n",
            "        [0.5730]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4070], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 4\n",
            "node feats are tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 1.4668e-03,  2.0452e-02,  1.2941e-02,  ..., -2.8648e-01,\n",
            "         -1.8963e-01, -5.9785e-02],\n",
            "        [ 7.5039e-03,  2.0582e-02,  4.8997e-03,  ..., -2.9289e-01,\n",
            "         -1.9638e-01, -6.1162e-02],\n",
            "        [ 3.4871e-03,  1.1607e-02,  3.7235e-03,  ..., -2.9779e-01,\n",
            "         -1.9579e-01, -5.9590e-02],\n",
            "        ...,\n",
            "        [ 3.9802e-03,  1.9151e-02,  3.3805e-03,  ..., -2.8445e-01,\n",
            "         -1.9262e-01, -5.3107e-02],\n",
            "        [-6.8497e-03,  1.4240e-02,  1.1740e-02,  ..., -2.9434e-01,\n",
            "         -1.9958e-01, -6.7843e-02],\n",
            "        [ 2.5627e-04,  1.7774e-02,  5.8555e-03,  ..., -2.8732e-01,\n",
            "         -2.0084e-01, -5.2453e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4541,  0.2626, -0.7614,  ..., -0.9501,  0.0514, -0.7373],\n",
            "        [-1.4507,  0.2531, -0.7570,  ..., -0.9467,  0.0475, -0.7306],\n",
            "        [-1.4530,  0.2500, -0.7556,  ..., -0.9417,  0.0534, -0.7373],\n",
            "        ...,\n",
            "        [-1.4604,  0.2616, -0.7621,  ..., -0.9485,  0.0413, -0.7277],\n",
            "        [-1.4592,  0.2508, -0.7605,  ..., -0.9500,  0.0521, -0.7381],\n",
            "        [-1.4561,  0.2493, -0.7604,  ..., -0.9528,  0.0526, -0.7382]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4011,  0.1095, -0.0170, -0.2608, -0.0560,  0.0758, -0.0417,  0.1539,\n",
            "          0.1429,  0.3887,  0.0663,  0.0106,  1.9401,  0.1959,  1.8908,  0.0644,\n",
            "          0.0802,  1.0889,  1.2717,  0.0000,  1.8122,  0.4041]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.0954, 0.0000,  ..., 0.0000, 1.7423, 0.3796],\n",
            "        [0.0000, 1.6839, 0.0000,  ..., 0.0000, 1.7884, 0.0000],\n",
            "        [0.0000, 2.0867, 0.0000,  ..., 0.0000, 1.7356, 0.3714],\n",
            "        ...,\n",
            "        [0.0000, 1.7091, 0.0000,  ..., 0.0000, 2.0305, 0.4307],\n",
            "        [0.0000, 2.4358, 0.0000,  ..., 0.0000, 2.5743, 0.6738],\n",
            "        [0.0000, 2.4343, 0.0000,  ..., 0.0000, 3.0199, 0.6012]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5571],\n",
            "        [0.5017],\n",
            "        [0.5564],\n",
            "        ...,\n",
            "        [0.5171],\n",
            "        [0.6176],\n",
            "        [0.5488]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5473],\n",
            "        [0.5473],\n",
            "        [0.5473],\n",
            "        ...,\n",
            "        [0.5473],\n",
            "        [0.5473],\n",
            "        [0.5473]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4122, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4122, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4122], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 5\n",
            "node feats are tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0090, -0.0119,  0.0392,  ..., -0.2794, -0.1795, -0.0391],\n",
            "        [-0.0092, -0.0106,  0.0261,  ..., -0.2764, -0.1803, -0.0314],\n",
            "        [-0.0077, -0.0130,  0.0346,  ..., -0.2769, -0.1781, -0.0322],\n",
            "        ...,\n",
            "        [-0.0139, -0.0085,  0.0324,  ..., -0.2882, -0.1807, -0.0357],\n",
            "        [-0.0059, -0.0112,  0.0248,  ..., -0.2819, -0.1740, -0.0423],\n",
            "        [-0.0110, -0.0079,  0.0312,  ..., -0.2693, -0.1806, -0.0361]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2745,  0.1780, -0.7524,  ..., -1.0516, -0.1050, -0.8107],\n",
            "        [-1.2872,  0.1879, -0.7584,  ..., -1.0613, -0.1149, -0.7995],\n",
            "        [-1.2731,  0.1816, -0.7449,  ..., -1.0484, -0.1155, -0.7918],\n",
            "        ...,\n",
            "        [-1.2816,  0.1883, -0.7541,  ..., -1.0581, -0.1118, -0.7970],\n",
            "        [-1.2794,  0.1783, -0.7484,  ..., -1.0556, -0.1085, -0.8042],\n",
            "        [-1.2754,  0.1896, -0.7519,  ..., -1.0519, -0.1140, -0.7919]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3711,  0.1242, -0.0145, -0.2539, -0.0469,  0.0755, -0.0691,  0.1785,\n",
            "          0.1377,  0.3929,  0.0030,  0.0020,  1.8605,  0.1496,  1.9593,  0.0923,\n",
            "          0.0424,  1.0703,  1.5195,  0.0000,  1.8046,  0.2745]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9761, 0.0059,  ..., 0.0000, 1.6920, 0.2935],\n",
            "        [0.0000, 1.8280, 0.0000,  ..., 0.0000, 1.2148, 0.0000],\n",
            "        [0.0000, 1.5601, 0.2215,  ..., 0.0000, 1.2429, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.7483, 0.0000,  ..., 0.0000, 1.6301, 0.3254],\n",
            "        [0.0000, 1.0841, 0.0000,  ..., 0.0000, 1.7212, 0.6416],\n",
            "        [0.0000, 1.9537, 0.0000,  ..., 0.0000, 1.7774, 0.1375]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5133],\n",
            "        [0.4246],\n",
            "        [0.4826],\n",
            "        ...,\n",
            "        [0.5270],\n",
            "        [0.5830],\n",
            "        [0.4689]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5694],\n",
            "        [0.5694],\n",
            "        [0.5694],\n",
            "        ...,\n",
            "        [0.5694],\n",
            "        [0.5694],\n",
            "        [0.5694]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4048, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4048, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4048], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 6\n",
            "node feats are tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0138,  0.0131,  0.0151,  ..., -0.2542, -0.1684, -0.0421],\n",
            "        [ 0.0157, -0.0057,  0.0136,  ..., -0.2522, -0.1601, -0.0511],\n",
            "        [ 0.0212, -0.0005,  0.0127,  ..., -0.2496, -0.1590, -0.0411],\n",
            "        ...,\n",
            "        [ 0.0182,  0.0020,  0.0092,  ..., -0.2484, -0.1678, -0.0478],\n",
            "        [ 0.0121,  0.0201,  0.0146,  ..., -0.2594, -0.1558, -0.0547],\n",
            "        [ 0.0175,  0.0067,  0.0144,  ..., -0.2480, -0.1676, -0.0486]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3047,  0.3010, -0.7712,  ..., -0.8691,  0.0300, -0.6989],\n",
            "        [-1.3182,  0.3099, -0.7764,  ..., -0.8774,  0.0302, -0.7014],\n",
            "        [-1.3066,  0.2984, -0.7680,  ..., -0.8688,  0.0257, -0.6975],\n",
            "        ...,\n",
            "        [-1.3062,  0.2905, -0.7728,  ..., -0.8674,  0.0269, -0.7005],\n",
            "        [-1.3166,  0.2996, -0.7797,  ..., -0.8810,  0.0312, -0.7109],\n",
            "        [-1.3020,  0.2859, -0.7684,  ..., -0.8713,  0.0308, -0.7034]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.0462e-01,  1.4141e-01, -1.5025e-02, -2.7260e-01, -5.5668e-02,\n",
            "          8.8621e-02, -8.3652e-02,  1.5320e-01,  1.5993e-01,  3.7042e-01,\n",
            "          1.6395e-02,  5.7187e-04,  1.9552e+00,  1.4649e-01,  2.0229e+00,\n",
            "          1.0578e-01,  7.1292e-02,  9.7411e-01,  1.3231e+00,  0.0000e+00,\n",
            "          2.1599e+00,  2.6845e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.7513, 0.0000,  ..., 0.0000, 2.2963, 0.8400],\n",
            "        [0.0000, 2.0942, 0.0000,  ..., 0.0000, 2.1448, 0.2135],\n",
            "        [0.0000, 1.7231, 0.2062,  ..., 0.0000, 1.6814, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.0925, 0.0000,  ..., 0.0000, 2.1382, 0.2104],\n",
            "        [0.0000, 1.8072, 0.0000,  ..., 0.0000, 2.0571, 0.1507],\n",
            "        [0.0000, 1.8632, 0.0000,  ..., 0.0000, 1.9963, 0.3287]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.8113],\n",
            "        [0.5170],\n",
            "        [0.4662],\n",
            "        ...,\n",
            "        [0.5163],\n",
            "        [0.4920],\n",
            "        [0.5633]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5825],\n",
            "        [0.5825],\n",
            "        [0.5825],\n",
            "        ...,\n",
            "        [0.5825],\n",
            "        [0.5825],\n",
            "        [0.5825]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4056, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4056, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4056], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 7\n",
            "node feats are tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0170,  0.0334, -0.0014,  ..., -0.2887, -0.1596, -0.0517],\n",
            "        [-0.0169,  0.0226,  0.0055,  ..., -0.2922, -0.1641, -0.0406],\n",
            "        [-0.0091,  0.0177, -0.0024,  ..., -0.2914, -0.1628, -0.0445],\n",
            "        ...,\n",
            "        [-0.0207,  0.0237,  0.0086,  ..., -0.2821, -0.1640, -0.0472],\n",
            "        [-0.0168,  0.0305,  0.0051,  ..., -0.2777, -0.1730, -0.0542],\n",
            "        [-0.0160,  0.0266, -0.0048,  ..., -0.2852, -0.1549, -0.0464]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3419e+00,  2.3156e-01, -7.8747e-01,  ..., -1.0851e+00,\n",
            "          5.3642e-03, -9.0167e-01],\n",
            "        [-1.3445e+00,  2.3571e-01, -7.8440e-01,  ..., -1.0855e+00,\n",
            "          3.2395e-03, -8.9941e-01],\n",
            "        [-1.3347e+00,  2.3677e-01, -7.7818e-01,  ..., -1.0820e+00,\n",
            "          4.1369e-03, -8.9475e-01],\n",
            "        ...,\n",
            "        [-1.3347e+00,  2.4783e-01, -7.8337e-01,  ..., -1.0811e+00,\n",
            "         -5.3877e-04, -8.8381e-01],\n",
            "        [-1.3402e+00,  2.3286e-01, -7.8068e-01,  ..., -1.0841e+00,\n",
            "          4.1263e-03, -9.0084e-01],\n",
            "        [-1.3358e+00,  2.3224e-01, -7.8378e-01,  ..., -1.0849e+00,\n",
            "          2.3689e-03, -8.9668e-01]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3560,  0.1287, -0.0156, -0.2515, -0.0466,  0.0741, -0.0677,  0.1936,\n",
            "          0.1381,  0.3638,  0.0370,  0.0115,  1.6487,  0.1701,  1.8837,  0.0982,\n",
            "          0.0528,  1.0794,  1.4702,  0.0000,  1.8050,  0.2681]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 0.4947, 0.8278,  ..., 0.0000, 2.5975, 0.3481],\n",
            "        [0.0000, 1.7043, 0.4614,  ..., 0.0000, 0.8831, 0.0000],\n",
            "        [0.0000, 1.7462, 0.0000,  ..., 0.0000, 1.7548, 0.1376],\n",
            "        ...,\n",
            "        [0.0000, 1.9077, 0.0243,  ..., 0.0000, 1.7417, 0.0076],\n",
            "        [0.0000, 1.1966, 0.2094,  ..., 0.0000, 1.4604, 0.2967],\n",
            "        [0.0000, 1.7732, 0.0000,  ..., 0.0000, 1.7569, 0.1901]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.2860],\n",
            "        [0.5409],\n",
            "        [0.4551],\n",
            "        ...,\n",
            "        [0.4114],\n",
            "        [0.4932],\n",
            "        [0.4728]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5676],\n",
            "        [0.5676],\n",
            "        [0.5676],\n",
            "        ...,\n",
            "        [0.5676],\n",
            "        [0.5676],\n",
            "        [0.5676]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4043, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4043, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4043], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 8\n",
            "node feats are tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.9340e-02, -5.2894e-03,  1.2369e-02,  ..., -2.6955e-01,\n",
            "         -1.7198e-01, -7.3454e-02],\n",
            "        [-2.7070e-02,  1.6359e-03,  9.9227e-03,  ..., -2.6343e-01,\n",
            "         -1.8509e-01, -6.8967e-02],\n",
            "        [-3.6096e-02, -1.2483e-02,  5.9846e-03,  ..., -2.6568e-01,\n",
            "         -1.8043e-01, -6.9604e-02],\n",
            "        ...,\n",
            "        [-3.5020e-02, -9.4335e-03,  1.2901e-02,  ..., -2.6308e-01,\n",
            "         -1.7988e-01, -6.2431e-02],\n",
            "        [-2.1236e-02, -1.1834e-02,  1.5079e-02,  ..., -2.7530e-01,\n",
            "         -1.7067e-01, -6.0995e-02],\n",
            "        [-3.2000e-02, -1.0553e-04,  1.2870e-02,  ..., -2.6883e-01,\n",
            "         -1.8371e-01, -6.7602e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2627,  0.1279, -0.5746,  ..., -0.9564,  0.0549, -0.6888],\n",
            "        [-1.2609,  0.1330, -0.5765,  ..., -0.9578,  0.0562, -0.6834],\n",
            "        [-1.2631,  0.1329, -0.5753,  ..., -0.9549,  0.0550, -0.6913],\n",
            "        ...,\n",
            "        [-1.2679,  0.1346, -0.5769,  ..., -0.9632,  0.0603, -0.6911],\n",
            "        [-1.2604,  0.1317, -0.5764,  ..., -0.9552,  0.0536, -0.6875],\n",
            "        [-1.2592,  0.1408, -0.5740,  ..., -0.9517,  0.0510, -0.6768]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3479,  0.1453, -0.0179, -0.2568, -0.0521,  0.0694, -0.0579,  0.2297,\n",
            "          0.1267,  0.3144,  0.1045,  0.0075,  1.4571,  0.3897,  1.7855,  0.1329,\n",
            "          0.0794,  1.1346,  1.3534,  0.0000,  1.8918,  0.2079]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.7531, 0.0000,  ..., 0.0000, 2.0111, 1.0674],\n",
            "        [0.0000, 0.9673, 1.3378,  ..., 0.0000, 2.2443, 0.0407],\n",
            "        [0.0144, 0.7248, 0.9012,  ..., 0.0000, 2.1668, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.0509, 0.6388,  ..., 0.0000, 1.9273, 0.0046],\n",
            "        [0.0000, 1.4583, 0.3874,  ..., 0.0000, 1.4722, 0.0000],\n",
            "        [0.0000, 1.9093, 0.1395,  ..., 0.0000, 1.8375, 1.1709]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.8484],\n",
            "        [0.3044],\n",
            "        [0.2982],\n",
            "        ...,\n",
            "        [0.5201],\n",
            "        [0.4719],\n",
            "        [0.7473]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5916],\n",
            "        [0.5916],\n",
            "        [0.5916],\n",
            "        ...,\n",
            "        [0.5916],\n",
            "        [0.5916],\n",
            "        [0.5916]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4025, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4025, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4025], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 9\n",
            "node feats are tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0009,  0.0536, -0.0117,  ..., -0.2549, -0.1838, -0.0700],\n",
            "        [ 0.0071,  0.0431, -0.0041,  ..., -0.2613, -0.1780, -0.0663],\n",
            "        [ 0.0096,  0.0492, -0.0125,  ..., -0.2597, -0.1766, -0.0710],\n",
            "        ...,\n",
            "        [ 0.0157,  0.0486, -0.0044,  ..., -0.2632, -0.1752, -0.0749],\n",
            "        [ 0.0212,  0.0418, -0.0126,  ..., -0.2577, -0.1759, -0.0588],\n",
            "        [ 0.0191,  0.0502, -0.0020,  ..., -0.2584, -0.1631, -0.0657]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3251,  0.0953, -0.7546,  ..., -0.8617,  0.0299, -0.6747],\n",
            "        [-1.3213,  0.1017, -0.7585,  ..., -0.8598,  0.0294, -0.6741],\n",
            "        [-1.3113,  0.0975, -0.7535,  ..., -0.8559,  0.0264, -0.6701],\n",
            "        ...,\n",
            "        [-1.3239,  0.1044, -0.7588,  ..., -0.8625,  0.0290, -0.6711],\n",
            "        [-1.3196,  0.1084, -0.7583,  ..., -0.8578,  0.0212, -0.6645],\n",
            "        [-1.3130,  0.0933, -0.7500,  ..., -0.8501,  0.0256, -0.6700]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3913,  0.1019, -0.0201, -0.2591, -0.0632,  0.0642, -0.0269,  0.1698,\n",
            "          0.1302,  0.3344,  0.1039,  0.0154,  1.7745,  0.2919,  1.6066,  0.0676,\n",
            "          0.1168,  1.1955,  1.1316,  0.0000,  1.8110,  0.2935]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1088, 0.1494,  ..., 0.0000, 1.3213, 0.1111],\n",
            "        [0.0000, 2.0006, 0.0529,  ..., 0.0000, 1.7974, 0.1993],\n",
            "        [0.0000, 1.7500, 0.0000,  ..., 0.0000, 1.7360, 0.0895],\n",
            "        ...,\n",
            "        [0.0000, 1.7160, 0.0428,  ..., 0.0000, 1.5423, 0.2039],\n",
            "        [0.0000, 1.8773, 0.0985,  ..., 0.0000, 1.6359, 0.1024],\n",
            "        [0.0000, 1.5239, 0.6399,  ..., 0.0000, 1.2788, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4891],\n",
            "        [0.4982],\n",
            "        [0.4956],\n",
            "        ...,\n",
            "        [0.5442],\n",
            "        [0.5621],\n",
            "        [0.4892]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5550],\n",
            "        [0.5550],\n",
            "        [0.5550],\n",
            "        ...,\n",
            "        [0.5550],\n",
            "        [0.5550],\n",
            "        [0.5550]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4070, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4070], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 10\n",
            "node feats are tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0091,  0.0154,  0.0242,  ..., -0.2576, -0.1869, -0.0489],\n",
            "        [-0.0163,  0.0236,  0.0164,  ..., -0.2526, -0.1839, -0.0623],\n",
            "        [-0.0047,  0.0246,  0.0236,  ..., -0.2556, -0.1786, -0.0487],\n",
            "        ...,\n",
            "        [-0.0060,  0.0245,  0.0137,  ..., -0.2536, -0.1863, -0.0466],\n",
            "        [-0.0126,  0.0198,  0.0158,  ..., -0.2582, -0.1800, -0.0535],\n",
            "        [-0.0075,  0.0264,  0.0235,  ..., -0.2516, -0.1903, -0.0540]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1809,  0.3088, -0.6944,  ..., -0.7849, -0.0682, -0.6329],\n",
            "        [-1.1809,  0.3107, -0.6937,  ..., -0.7866, -0.0701, -0.6296],\n",
            "        [-1.1920,  0.3139, -0.6979,  ..., -0.7931, -0.0739, -0.6329],\n",
            "        ...,\n",
            "        [-1.1834,  0.2970, -0.6966,  ..., -0.7930, -0.0631, -0.6398],\n",
            "        [-1.1869,  0.3066, -0.6947,  ..., -0.7904, -0.0578, -0.6478],\n",
            "        [-1.1866,  0.3103, -0.6973,  ..., -0.7894, -0.0673, -0.6361]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4082,  0.1161, -0.0181, -0.2733, -0.0521,  0.0880, -0.0419,  0.1724,\n",
            "          0.1519,  0.3796,  0.0854,  0.0035,  1.9333,  0.2777,  2.0687,  0.0739,\n",
            "          0.1068,  1.0240,  1.1240,  0.0000,  1.8430,  0.2932]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.5619, 0.3958,  ..., 0.0000, 1.8640, 0.1462],\n",
            "        [0.0000, 2.1925, 0.2568,  ..., 0.0000, 1.7814, 0.0000],\n",
            "        [0.0000, 2.1250, 0.0668,  ..., 0.0000, 1.8303, 0.2289],\n",
            "        ...,\n",
            "        [0.0000, 1.5853, 0.0820,  ..., 0.0000, 1.7177, 0.0000],\n",
            "        [0.0000, 2.0313, 0.1961,  ..., 0.0000, 1.8025, 0.2552],\n",
            "        [0.0000, 1.6957, 0.5443,  ..., 0.0000, 1.4196, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.6355],\n",
            "        [0.5098],\n",
            "        [0.5624],\n",
            "        ...,\n",
            "        [0.4528],\n",
            "        [0.5735],\n",
            "        [0.5325]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5719],\n",
            "        [0.5719],\n",
            "        [0.5719],\n",
            "        ...,\n",
            "        [0.5719],\n",
            "        [0.5719],\n",
            "        [0.5719]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4101, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4101, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4101], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 11\n",
            "node feats are tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0117,  0.0011,  0.0115,  ..., -0.2776, -0.1770, -0.0320],\n",
            "        [ 0.0078, -0.0065,  0.0217,  ..., -0.2831, -0.1879, -0.0436],\n",
            "        [ 0.0017, -0.0090,  0.0161,  ..., -0.2887, -0.1846, -0.0283],\n",
            "        ...,\n",
            "        [ 0.0102, -0.0105,  0.0239,  ..., -0.2792, -0.1860, -0.0408],\n",
            "        [ 0.0045, -0.0128,  0.0167,  ..., -0.2849, -0.1991, -0.0409],\n",
            "        [ 0.0108, -0.0092,  0.0222,  ..., -0.2808, -0.1868, -0.0339]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3264,  0.2730, -0.8160,  ..., -1.0439, -0.1009, -0.7811],\n",
            "        [-1.3329,  0.2783, -0.8199,  ..., -1.0494, -0.1005, -0.7840],\n",
            "        [-1.3371,  0.2896, -0.8219,  ..., -1.0497, -0.1035, -0.7762],\n",
            "        ...,\n",
            "        [-1.3393,  0.2904, -0.8245,  ..., -1.0535, -0.1028, -0.7844],\n",
            "        [-1.3288,  0.2842, -0.8162,  ..., -1.0465, -0.1036, -0.7776],\n",
            "        [-1.3312,  0.2752, -0.8163,  ..., -1.0462, -0.1008, -0.7849]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3617,  0.1142, -0.0129, -0.2466, -0.0459,  0.0640, -0.0720,  0.1729,\n",
            "          0.1234,  0.3932, -0.0548,  0.0073,  1.9283,  0.1311,  1.7546,  0.0696,\n",
            "          0.0226,  1.1148,  1.6416,  0.0000,  1.7420,  0.1742]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1555, 0.0000,  ..., 0.0000, 1.4630, 0.0000],\n",
            "        [0.0000, 2.0645, 0.0000,  ..., 0.0000, 1.6804, 0.0651],\n",
            "        [0.0000, 1.3689, 0.1044,  ..., 0.0000, 0.7979, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.0760, 0.0000,  ..., 0.0000, 1.6840, 0.0655],\n",
            "        [0.0000, 1.7907, 0.0000,  ..., 0.0000, 1.5134, 0.1706],\n",
            "        [0.0000, 1.5747, 0.8215,  ..., 0.0000, 2.1891, 0.1064]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4344],\n",
            "        [0.4473],\n",
            "        [0.4681],\n",
            "        ...,\n",
            "        [0.4485],\n",
            "        [0.4886],\n",
            "        [0.3267]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5779],\n",
            "        [0.5779],\n",
            "        [0.5779],\n",
            "        ...,\n",
            "        [0.5779],\n",
            "        [0.5779],\n",
            "        [0.5779]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4034, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4034, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4034], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 12\n",
            "node feats are tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0114,  0.0140,  0.0201,  ..., -0.2715, -0.1943, -0.0629],\n",
            "        [-0.0076,  0.0139,  0.0199,  ..., -0.2806, -0.1836, -0.0628],\n",
            "        [-0.0104,  0.0073,  0.0229,  ..., -0.2805, -0.1873, -0.0601],\n",
            "        ...,\n",
            "        [-0.0247,  0.0142,  0.0107,  ..., -0.2787, -0.1918, -0.0588],\n",
            "        [-0.0125,  0.0101,  0.0198,  ..., -0.2729, -0.1909, -0.0623],\n",
            "        [-0.0082,  0.0175,  0.0223,  ..., -0.2829, -0.1906, -0.0583]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4338,  0.1877, -0.9183,  ..., -1.0143,  0.1504, -0.8645],\n",
            "        [-1.4339,  0.1682, -0.9227,  ..., -1.0225,  0.1620, -0.8865],\n",
            "        [-1.4243,  0.1706, -0.9146,  ..., -1.0144,  0.1575, -0.8685],\n",
            "        ...,\n",
            "        [-1.4332,  0.1737, -0.9142,  ..., -1.0187,  0.1549, -0.8732],\n",
            "        [-1.4368,  0.1760, -0.9182,  ..., -1.0170,  0.1550, -0.8716],\n",
            "        [-1.4216,  0.1793, -0.9116,  ..., -1.0125,  0.1570, -0.8664]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.1257e-01,  1.2883e-01, -1.5650e-02, -2.7548e-01, -5.0893e-02,\n",
            "          9.1586e-02, -7.0061e-02,  1.4328e-01,  1.6754e-01,  3.8462e-01,\n",
            "          1.0450e-02,  2.0564e-03,  2.0258e+00,  1.1644e-01,  2.0671e+00,\n",
            "          3.5442e-02,  9.9393e-02,  8.9019e-01,  1.2284e+00,  0.0000e+00,\n",
            "          2.0117e+00,  2.8033e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.4922, 0.0000,  ..., 0.0000, 1.6834, 0.0000],\n",
            "        [0.0000, 2.1468, 0.0000,  ..., 0.0000, 2.3528, 0.4638],\n",
            "        [0.0000, 2.2720, 0.0000,  ..., 0.0000, 1.7517, 0.0868],\n",
            "        ...,\n",
            "        [0.0000, 1.8452, 0.0000,  ..., 0.0000, 1.7337, 0.4031],\n",
            "        [0.0000, 2.1689, 0.0000,  ..., 0.0000, 1.8530, 0.2008],\n",
            "        [0.0000, 2.2502, 0.0000,  ..., 0.0000, 2.2252, 0.5158]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4055],\n",
            "        [0.5640],\n",
            "        [0.5215],\n",
            "        ...,\n",
            "        [0.6229],\n",
            "        [0.5473],\n",
            "        [0.5621]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5761],\n",
            "        [0.5761],\n",
            "        [0.5761],\n",
            "        ...,\n",
            "        [0.5761],\n",
            "        [0.5761],\n",
            "        [0.5761]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4104, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4104, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4104], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 13\n",
            "node feats are tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0049,  0.0048,  0.0239,  ..., -0.2752, -0.1616, -0.0598],\n",
            "        [ 0.0038,  0.0095,  0.0226,  ..., -0.2795, -0.1733, -0.0630],\n",
            "        [-0.0010,  0.0054,  0.0148,  ..., -0.2782, -0.1634, -0.0572],\n",
            "        ...,\n",
            "        [ 0.0069,  0.0016,  0.0178,  ..., -0.2774, -0.1621, -0.0526],\n",
            "        [ 0.0056,  0.0021,  0.0159,  ..., -0.2780, -0.1550, -0.0486],\n",
            "        [ 0.0105,  0.0015,  0.0100,  ..., -0.2739, -0.1579, -0.0582]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3209,  0.1288, -0.6916,  ..., -0.9948,  0.0305, -0.7889],\n",
            "        [-1.3249,  0.1212, -0.6978,  ..., -0.9978,  0.0395, -0.8116],\n",
            "        [-1.3238,  0.1215, -0.6903,  ..., -0.9907,  0.0350, -0.7945],\n",
            "        ...,\n",
            "        [-1.3265,  0.1245, -0.6981,  ..., -0.9987,  0.0387, -0.8012],\n",
            "        [-1.3269,  0.1213, -0.6955,  ..., -0.9962,  0.0379, -0.8033],\n",
            "        [-1.3256,  0.1218, -0.6968,  ..., -0.9962,  0.0366, -0.8027]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3691,  0.1308, -0.0150, -0.2544, -0.0521,  0.0653, -0.0656,  0.1857,\n",
            "          0.1277,  0.3687,  0.0146,  0.0059,  1.7967,  0.2517,  1.7728,  0.0651,\n",
            "          0.0585,  1.1320,  1.5146,  0.0000,  1.8807,  0.2481]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.5991, 0.8517,  ..., 0.0000, 2.4315, 0.1968],\n",
            "        [0.0000, 2.9221, 0.0000,  ..., 0.0000, 2.0722, 1.0120],\n",
            "        [0.0000, 2.0383, 0.1585,  ..., 0.0000, 1.8091, 0.0392],\n",
            "        ...,\n",
            "        [0.0000, 0.9820, 1.2315,  ..., 0.0000, 2.5900, 0.0719],\n",
            "        [0.0000, 1.9265, 0.0776,  ..., 0.0000, 1.8227, 0.1877],\n",
            "        [0.0000, 1.9300, 0.0747,  ..., 0.0000, 1.8319, 0.1855]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.2264],\n",
            "        [0.7866],\n",
            "        [0.4315],\n",
            "        ...,\n",
            "        [0.2334],\n",
            "        [0.4680],\n",
            "        [0.4680]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5806],\n",
            "        [0.5806],\n",
            "        [0.5806],\n",
            "        ...,\n",
            "        [0.5806],\n",
            "        [0.5806],\n",
            "        [0.5806]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4012, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4012, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4012], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 14\n",
            "node feats are tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0113,  0.0121, -0.0081,  ..., -0.2690, -0.1716, -0.0557],\n",
            "        [-0.0145,  0.0233, -0.0079,  ..., -0.2698, -0.1719, -0.0641],\n",
            "        [-0.0037,  0.0196, -0.0118,  ..., -0.2596, -0.1651, -0.0662],\n",
            "        ...,\n",
            "        [-0.0096,  0.0202, -0.0084,  ..., -0.2670, -0.1727, -0.0670],\n",
            "        [-0.0072,  0.0251, -0.0006,  ..., -0.2635, -0.1678, -0.0647],\n",
            "        [-0.0158,  0.0167, -0.0050,  ..., -0.2753, -0.1685, -0.0610]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3547,  0.2342, -0.8716,  ..., -0.9276, -0.0027, -0.7817],\n",
            "        [-1.3727,  0.2425, -0.8876,  ..., -0.9391, -0.0046, -0.7957],\n",
            "        [-1.3519,  0.2345, -0.8696,  ..., -0.9286, -0.0079, -0.7813],\n",
            "        ...,\n",
            "        [-1.3575,  0.2364, -0.8729,  ..., -0.9289, -0.0064, -0.7783],\n",
            "        [-1.3553,  0.2325, -0.8748,  ..., -0.9294, -0.0056, -0.7806],\n",
            "        [-1.3540,  0.2457, -0.8688,  ..., -0.9254, -0.0125, -0.7665]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3917,  0.1217, -0.0168, -0.2658, -0.0524,  0.0830, -0.0596,  0.1641,\n",
            "          0.1500,  0.3688,  0.0445,  0.0050,  1.8575,  0.1696,  1.9293,  0.0848,\n",
            "          0.0702,  1.0142,  1.2576,  0.0000,  1.8929,  0.2770]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1243, 0.0000,  ..., 0.0000, 1.6995, 0.1553],\n",
            "        [0.0000, 2.0332, 0.0000,  ..., 0.0000, 1.8973, 0.1350],\n",
            "        [0.0000, 1.9876, 0.0174,  ..., 0.0000, 1.8493, 0.2391],\n",
            "        ...,\n",
            "        [0.0000, 1.3783, 0.2785,  ..., 0.0000, 1.0125, 0.0000],\n",
            "        [0.0000, 1.3128, 0.2802,  ..., 0.0000, 1.5629, 0.3769],\n",
            "        [0.0000, 2.0398, 0.0000,  ..., 0.0000, 1.8244, 0.1493]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.6491],\n",
            "        [0.5130],\n",
            "        [0.5328],\n",
            "        ...,\n",
            "        [0.5023],\n",
            "        [0.5243],\n",
            "        [0.5057]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5668],\n",
            "        [0.5668],\n",
            "        [0.5668],\n",
            "        ...,\n",
            "        [0.5668],\n",
            "        [0.5668],\n",
            "        [0.5668]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4064, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4064, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4064], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 15\n",
            "node feats are tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0278,  0.0103,  0.0129,  ..., -0.2573, -0.1668, -0.0710],\n",
            "        [ 0.0261,  0.0178,  0.0129,  ..., -0.2684, -0.1686, -0.0701],\n",
            "        [ 0.0296,  0.0165,  0.0066,  ..., -0.2647, -0.1786, -0.0608],\n",
            "        ...,\n",
            "        [ 0.0225,  0.0174,  0.0086,  ..., -0.2641, -0.1710, -0.0646],\n",
            "        [ 0.0232,  0.0127,  0.0141,  ..., -0.2590, -0.1666, -0.0686],\n",
            "        [ 0.0277,  0.0235,  0.0146,  ..., -0.2654, -0.1706, -0.0685]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3990,  0.2379, -0.8188,  ..., -0.9269,  0.0232, -0.7641],\n",
            "        [-1.3841,  0.2357, -0.8160,  ..., -0.9194,  0.0274, -0.7607],\n",
            "        [-1.3905,  0.2340, -0.8137,  ..., -0.9242,  0.0220, -0.7579],\n",
            "        ...,\n",
            "        [-1.3999,  0.2362, -0.8183,  ..., -0.9244,  0.0230, -0.7578],\n",
            "        [-1.3923,  0.2562, -0.8187,  ..., -0.9249,  0.0134, -0.7474],\n",
            "        [-1.3883,  0.2421, -0.8150,  ..., -0.9208,  0.0270, -0.7568]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4065,  0.1222, -0.0162, -0.2665, -0.0578,  0.0839, -0.0609,  0.1520,\n",
            "          0.1533,  0.3833,  0.0656,  0.0052,  1.9494,  0.1611,  2.0086,  0.1207,\n",
            "          0.0726,  1.0645,  1.2715,  0.0000,  1.9992,  0.3942]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.0100, 0.0000,  ..., 0.0000, 1.8386, 0.5066],\n",
            "        [0.0000, 2.5541, 0.1602,  ..., 0.0000, 2.0624, 0.1606],\n",
            "        [0.0000, 1.2191, 0.0000,  ..., 0.0000, 1.6426, 0.5083],\n",
            "        ...,\n",
            "        [0.0000, 2.0642, 0.0000,  ..., 0.0000, 1.9479, 0.3445],\n",
            "        [0.0000, 2.1484, 0.1341,  ..., 0.0000, 2.4289, 0.3796],\n",
            "        [0.0000, 1.5668, 0.7744,  ..., 0.0000, 1.1061, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5762],\n",
            "        [0.5987],\n",
            "        [0.5160],\n",
            "        ...,\n",
            "        [0.5474],\n",
            "        [0.6060],\n",
            "        [0.5828]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5540],\n",
            "        [0.5540],\n",
            "        [0.5540],\n",
            "        ...,\n",
            "        [0.5540],\n",
            "        [0.5540],\n",
            "        [0.5540]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4107, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4107, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4107], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 16\n",
            "node feats are tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0090,  0.0246,  0.0091,  ..., -0.2840, -0.1601, -0.0525],\n",
            "        [-0.0096,  0.0247,  0.0049,  ..., -0.2849, -0.1611, -0.0519],\n",
            "        [-0.0042,  0.0197,  0.0128,  ..., -0.2680, -0.1594, -0.0507],\n",
            "        ...,\n",
            "        [-0.0128,  0.0144,  0.0157,  ..., -0.2806, -0.1577, -0.0565],\n",
            "        [-0.0093,  0.0282,  0.0085,  ..., -0.2708, -0.1707, -0.0534],\n",
            "        [-0.0124,  0.0255,  0.0050,  ..., -0.2667, -0.1543, -0.0599]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3322,  0.1192, -0.7541,  ..., -1.0111,  0.0670, -0.8312],\n",
            "        [-1.3401,  0.1221, -0.7558,  ..., -1.0162,  0.0612, -0.8223],\n",
            "        [-1.3386,  0.1212, -0.7556,  ..., -1.0114,  0.0682, -0.8296],\n",
            "        ...,\n",
            "        [-1.3374,  0.1133, -0.7517,  ..., -1.0113,  0.0669, -0.8301],\n",
            "        [-1.3467,  0.1380, -0.7667,  ..., -1.0220,  0.0579, -0.8231],\n",
            "        [-1.3508,  0.1247, -0.7638,  ..., -1.0215,  0.0681, -0.8319]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3994,  0.1095, -0.0183, -0.2647, -0.0528,  0.0822, -0.0524,  0.1724,\n",
            "          0.1565,  0.3669,  0.0746,  0.0057,  1.8285,  0.1623,  1.9963,  0.0483,\n",
            "          0.1315,  1.1058,  1.1899,  0.0000,  1.8992,  0.2983]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9528, 0.0000,  ..., 0.0000, 1.8876, 0.2041],\n",
            "        [0.0000, 1.4186, 0.0000,  ..., 0.0000, 2.1842, 0.4477],\n",
            "        [0.0000, 2.5288, 0.5572,  ..., 0.0000, 2.8088, 1.0630],\n",
            "        ...,\n",
            "        [0.0000, 1.2148, 0.0000,  ..., 0.0000, 1.8310, 0.0000],\n",
            "        [0.0000, 1.7024, 0.0000,  ..., 0.0000, 1.6778, 0.6757],\n",
            "        [0.0000, 2.8979, 0.2214,  ..., 0.0000, 1.4424, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5206],\n",
            "        [0.5233],\n",
            "        [0.6788],\n",
            "        ...,\n",
            "        [0.3982],\n",
            "        [0.6500],\n",
            "        [0.5505]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5576],\n",
            "        [0.5576],\n",
            "        [0.5576],\n",
            "        ...,\n",
            "        [0.5576],\n",
            "        [0.5576],\n",
            "        [0.5576]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4095, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4095, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4095], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 17\n",
            "node feats are tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0079,  0.0254,  0.0181,  ..., -0.2632, -0.1752, -0.0748],\n",
            "        [-0.0134,  0.0167,  0.0219,  ..., -0.2746, -0.1871, -0.0793],\n",
            "        [ 0.0019,  0.0174,  0.0151,  ..., -0.2743, -0.1816, -0.0804],\n",
            "        ...,\n",
            "        [-0.0037,  0.0207,  0.0091,  ..., -0.2715, -0.1778, -0.0823],\n",
            "        [-0.0103,  0.0088,  0.0067,  ..., -0.2611, -0.1777, -0.0785],\n",
            "        [-0.0106,  0.0212,  0.0280,  ..., -0.2664, -0.1819, -0.0766]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1957,  0.2064, -0.6308,  ..., -0.7900,  0.1073, -0.6685],\n",
            "        [-1.1948,  0.1942, -0.6329,  ..., -0.7897,  0.1104, -0.6832],\n",
            "        [-1.1995,  0.2004, -0.6376,  ..., -0.7940,  0.1102, -0.6762],\n",
            "        ...,\n",
            "        [-1.1882,  0.2004, -0.6275,  ..., -0.7859,  0.1046, -0.6645],\n",
            "        [-1.1872,  0.1902, -0.6270,  ..., -0.7810,  0.1092, -0.6745],\n",
            "        [-1.1999,  0.2050, -0.6366,  ..., -0.7920,  0.1033, -0.6703]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3741,  0.1412, -0.0145, -0.2603, -0.0493,  0.0703, -0.0728,  0.1966,\n",
            "          0.1294,  0.3660,  0.0051,  0.0121,  1.8465,  0.3160,  1.8443,  0.0535,\n",
            "          0.0943,  1.0859,  1.5154,  0.0000,  1.9444,  0.1461]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.2561, 0.4509,  ..., 0.0000, 1.5049, 0.1764],\n",
            "        [0.0000, 1.0914, 0.0127,  ..., 0.0000, 1.8351, 0.0000],\n",
            "        [0.0000, 2.0569, 0.0000,  ..., 0.0000, 2.3636, 0.3289],\n",
            "        ...,\n",
            "        [0.0000, 1.5440, 0.0000,  ..., 0.0000, 1.8983, 0.2131],\n",
            "        [0.0000, 1.8738, 0.4305,  ..., 0.0000, 1.0852, 0.0000],\n",
            "        [0.3979, 0.0000, 1.5454,  ..., 0.0000, 2.1180, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5051],\n",
            "        [0.3311],\n",
            "        [0.4357],\n",
            "        ...,\n",
            "        [0.4971],\n",
            "        [0.4578],\n",
            "        [0.3341]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.6087],\n",
            "        [0.6087],\n",
            "        [0.6087],\n",
            "        ...,\n",
            "        [0.6087],\n",
            "        [0.6087],\n",
            "        [0.6087]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4000, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4000, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4000], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 18\n",
            "node feats are tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0094, -0.0164,  0.0193,  ..., -0.2538, -0.1558, -0.0238],\n",
            "        [-0.0034, -0.0147,  0.0240,  ..., -0.2540, -0.1556, -0.0252],\n",
            "        [-0.0078, -0.0077,  0.0195,  ..., -0.2567, -0.1541, -0.0325],\n",
            "        ...,\n",
            "        [-0.0102, -0.0150,  0.0177,  ..., -0.2604, -0.1592, -0.0290],\n",
            "        [-0.0151, -0.0070,  0.0267,  ..., -0.2690, -0.1522, -0.0284],\n",
            "        [-0.0028, -0.0200,  0.0167,  ..., -0.2601, -0.1590, -0.0255]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3625,  0.1890, -0.8592,  ..., -1.2076,  0.0729, -0.9018],\n",
            "        [-1.3606,  0.1793, -0.8558,  ..., -1.2038,  0.0774, -0.9045],\n",
            "        [-1.3667,  0.1702, -0.8631,  ..., -1.2125,  0.0800, -0.9218],\n",
            "        ...,\n",
            "        [-1.3628,  0.1833, -0.8601,  ..., -1.2082,  0.0781, -0.9133],\n",
            "        [-1.3662,  0.1782, -0.8590,  ..., -1.2094,  0.0739, -0.9071],\n",
            "        [-1.3594,  0.1706, -0.8582,  ..., -1.2064,  0.0816, -0.9217]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3624,  0.1505, -0.0129, -0.2518, -0.0477,  0.0755, -0.0890,  0.1837,\n",
            "          0.1414,  0.3846, -0.0070,  0.0037,  1.7612,  0.1420,  1.9800,  0.0959,\n",
            "          0.0436,  1.0324,  1.6477,  0.0000,  1.9720,  0.3188]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.6803, 0.0000,  ..., 0.0000, 1.8320, 0.0000],\n",
            "        [0.0000, 1.3289, 0.0398,  ..., 0.0000, 1.6291, 0.3959],\n",
            "        [0.0000, 2.7872, 0.0000,  ..., 0.0000, 2.1295, 1.0587],\n",
            "        ...,\n",
            "        [0.0000, 1.4134, 0.0000,  ..., 0.0000, 2.2892, 0.4530],\n",
            "        [0.0000, 1.8928, 0.0000,  ..., 0.0000, 1.9182, 0.2653],\n",
            "        [0.0000, 1.7782, 0.0000,  ..., 0.0000, 2.0984, 0.0565]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4573],\n",
            "        [0.4893],\n",
            "        [0.7522],\n",
            "        ...,\n",
            "        [0.4551],\n",
            "        [0.4700],\n",
            "        [0.4029]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5774],\n",
            "        [0.5774],\n",
            "        [0.5774],\n",
            "        ...,\n",
            "        [0.5774],\n",
            "        [0.5774],\n",
            "        [0.5774]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4041], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 19\n",
            "node feats are tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0100, -0.0176,  0.0231,  ..., -0.2758, -0.1593, -0.0689],\n",
            "        [-0.0261, -0.0093,  0.0215,  ..., -0.2659, -0.1696, -0.0528],\n",
            "        [-0.0217, -0.0101,  0.0259,  ..., -0.2765, -0.1694, -0.0468],\n",
            "        ...,\n",
            "        [-0.0268, -0.0049,  0.0257,  ..., -0.2746, -0.1596, -0.0533],\n",
            "        [-0.0230, -0.0170,  0.0276,  ..., -0.2640, -0.1665, -0.0533],\n",
            "        [-0.0113, -0.0153,  0.0111,  ..., -0.2769, -0.1676, -0.0621]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4009,  0.1995, -0.7530,  ..., -1.1004,  0.1628, -0.8131],\n",
            "        [-1.3987,  0.1826, -0.7454,  ..., -1.0929,  0.1609, -0.8210],\n",
            "        [-1.3940,  0.1825, -0.7470,  ..., -1.0930,  0.1674, -0.8253],\n",
            "        ...,\n",
            "        [-1.3973,  0.1916, -0.7519,  ..., -1.0942,  0.1624, -0.8176],\n",
            "        [-1.3983,  0.1940, -0.7441,  ..., -1.0905,  0.1552, -0.8070],\n",
            "        [-1.3907,  0.2017, -0.7461,  ..., -1.0890,  0.1579, -0.8050]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 3.7805e-01,  1.5326e-01, -1.5793e-02, -2.6720e-01, -5.2526e-02,\n",
            "          8.6762e-02, -8.9264e-02,  1.7763e-01,  1.5886e-01,  3.3632e-01,\n",
            "          3.0280e-02,  7.0680e-04,  1.6790e+00,  1.6058e-01,  1.9429e+00,\n",
            "          9.6470e-02,  7.1868e-02,  9.5029e-01,  1.3388e+00,  0.0000e+00,\n",
            "          2.1428e+00,  2.2796e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.2481, 0.0613,  ..., 0.0000, 1.7567, 0.1813],\n",
            "        [0.0000, 1.6035, 0.0000,  ..., 0.0000, 1.7845, 0.3783],\n",
            "        [0.0000, 1.8190, 0.0409,  ..., 0.0000, 1.9890, 0.3183],\n",
            "        ...,\n",
            "        [0.0000, 1.5620, 0.0000,  ..., 0.0000, 2.0597, 0.0000],\n",
            "        [0.0000, 1.7955, 0.0000,  ..., 0.0000, 2.1229, 0.1472],\n",
            "        [0.0000, 1.9520, 0.0000,  ..., 0.0000, 2.0951, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4716],\n",
            "        [0.5253],\n",
            "        [0.5270],\n",
            "        ...,\n",
            "        [0.4813],\n",
            "        [0.4746],\n",
            "        [0.4205]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5903],\n",
            "        [0.5903],\n",
            "        [0.5903],\n",
            "        ...,\n",
            "        [0.5903],\n",
            "        [0.5903],\n",
            "        [0.5903]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4041], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "===== Epoch 4, Loss tensor([28.1243], grad_fn=<DivBackward0>) =====\n",
            "index of dataloader is 0\n",
            "node feats are tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0131,  0.0246,  0.0147,  ..., -0.2711, -0.1619, -0.0371],\n",
            "        [-0.0110,  0.0191,  0.0101,  ..., -0.2670, -0.1643, -0.0325],\n",
            "        [-0.0168,  0.0219,  0.0122,  ..., -0.2715, -0.1535, -0.0410],\n",
            "        ...,\n",
            "        [-0.0184,  0.0240,  0.0143,  ..., -0.2700, -0.1634, -0.0304],\n",
            "        [-0.0141,  0.0301,  0.0259,  ..., -0.2668, -0.1545, -0.0318],\n",
            "        [-0.0121,  0.0390,  0.0163,  ..., -0.2709, -0.1652, -0.0400]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3427,  0.2005, -0.8687,  ..., -1.0882,  0.0151, -0.9812],\n",
            "        [-1.3322,  0.1918, -0.8604,  ..., -1.0743,  0.0156, -0.9795],\n",
            "        [-1.3413,  0.1981, -0.8627,  ..., -1.0801,  0.0106, -0.9701],\n",
            "        ...,\n",
            "        [-1.3396,  0.1943, -0.8669,  ..., -1.0825,  0.0177, -0.9878],\n",
            "        [-1.3382,  0.2010, -0.8658,  ..., -1.0793,  0.0124, -0.9734],\n",
            "        [-1.3311,  0.1982, -0.8586,  ..., -1.0722,  0.0110, -0.9668]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3718,  0.1307, -0.0145, -0.2544, -0.0473,  0.0760, -0.0727,  0.1819,\n",
            "          0.1413,  0.3912,  0.0127,  0.0073,  1.8194,  0.1604,  1.9960,  0.0809,\n",
            "          0.0537,  1.0753,  1.5358,  0.0000,  1.8666,  0.3077]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.5367, 0.0000,  ..., 0.0000, 1.5275, 0.0000],\n",
            "        [0.0000, 2.8597, 0.0000,  ..., 0.0000, 1.9781, 1.0959],\n",
            "        [0.0000, 1.9919, 0.0000,  ..., 0.0000, 1.8168, 0.3810],\n",
            "        ...,\n",
            "        [0.0000, 3.6228, 0.0000,  ..., 0.0000, 1.9432, 0.9330],\n",
            "        [0.0000, 1.9598, 0.0000,  ..., 0.0000, 1.7948, 0.2829],\n",
            "        [0.0000, 1.3593, 0.2187,  ..., 0.0000, 1.5387, 0.3657]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5023],\n",
            "        [0.7858],\n",
            "        [0.5571],\n",
            "        ...,\n",
            "        [0.8319],\n",
            "        [0.5044],\n",
            "        [0.5161]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5699],\n",
            "        [0.5699],\n",
            "        [0.5699],\n",
            "        ...,\n",
            "        [0.5699],\n",
            "        [0.5699],\n",
            "        [0.5699]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4077, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4077, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4077], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 1\n",
            "node feats are tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0039, -0.0168,  0.0093,  ..., -0.2757, -0.1949, -0.0369],\n",
            "        [ 0.0033, -0.0213,  0.0006,  ..., -0.2852, -0.1920, -0.0365],\n",
            "        [-0.0024, -0.0021, -0.0005,  ..., -0.2720, -0.1891, -0.0360],\n",
            "        ...,\n",
            "        [ 0.0058, -0.0158, -0.0043,  ..., -0.2828, -0.1897, -0.0368],\n",
            "        [-0.0068, -0.0130, -0.0089,  ..., -0.2835, -0.1958, -0.0399],\n",
            "        [-0.0006, -0.0234,  0.0013,  ..., -0.2810, -0.1919, -0.0394]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4916,  0.1024, -0.7639,  ..., -1.1170,  0.1129, -0.7971],\n",
            "        [-1.5019,  0.1051, -0.7706,  ..., -1.1264,  0.1142, -0.8000],\n",
            "        [-1.4965,  0.0950, -0.7599,  ..., -1.1230,  0.1117, -0.7928],\n",
            "        ...,\n",
            "        [-1.5027,  0.1129, -0.7704,  ..., -1.1335,  0.1126, -0.7962],\n",
            "        [-1.5111,  0.1108, -0.7744,  ..., -1.1340,  0.1152, -0.8029],\n",
            "        [-1.4985,  0.0945, -0.7696,  ..., -1.1265,  0.1204, -0.8061]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3724,  0.1150, -0.0170, -0.2519, -0.0525,  0.0700, -0.0634,  0.1860,\n",
            "          0.1420,  0.3645,  0.0517,  0.0062,  1.7013,  0.1470,  1.8792,  0.0686,\n",
            "          0.0872,  1.2030,  1.4193,  0.0000,  1.8932,  0.3182]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 0.6008, 0.4134,  ..., 0.0000, 0.9194, 0.0137],\n",
            "        [0.0000, 1.7138, 0.0000,  ..., 0.0000, 2.1245, 0.4373],\n",
            "        [0.0000, 1.8223, 0.0000,  ..., 0.0000, 1.8255, 0.2493],\n",
            "        ...,\n",
            "        [0.0000, 1.2386, 0.9439,  ..., 0.0000, 2.4506, 0.1477],\n",
            "        [0.0000, 1.2930, 0.0000,  ..., 0.0000, 2.0972, 0.5517],\n",
            "        [0.0000, 1.9854, 0.0000,  ..., 0.0000, 1.5922, 0.1098]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4458],\n",
            "        [0.4538],\n",
            "        [0.4781],\n",
            "        ...,\n",
            "        [0.2672],\n",
            "        [0.5063],\n",
            "        [0.4491]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5541],\n",
            "        [0.5541],\n",
            "        [0.5541],\n",
            "        ...,\n",
            "        [0.5541],\n",
            "        [0.5541],\n",
            "        [0.5541]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4088, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4088, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4088], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 2\n",
            "node feats are tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0368,  0.0102,  0.0214,  ..., -0.2598, -0.1549, -0.0510],\n",
            "        [ 0.0409, -0.0075,  0.0189,  ..., -0.2633, -0.1544, -0.0473],\n",
            "        [ 0.0311, -0.0005,  0.0155,  ..., -0.2623, -0.1562, -0.0533],\n",
            "        ...,\n",
            "        [ 0.0417,  0.0033,  0.0156,  ..., -0.2648, -0.1490, -0.0481],\n",
            "        [ 0.0419,  0.0013,  0.0142,  ..., -0.2688, -0.1486, -0.0521],\n",
            "        [ 0.0413,  0.0031,  0.0093,  ..., -0.2693, -0.1470, -0.0547]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4004,  0.3377, -0.7637,  ..., -0.9403,  0.0842, -0.6457],\n",
            "        [-1.3951,  0.3374, -0.7605,  ..., -0.9305,  0.0786, -0.6376],\n",
            "        [-1.3937,  0.3479, -0.7608,  ..., -0.9349,  0.0772, -0.6295],\n",
            "        ...,\n",
            "        [-1.4011,  0.3344, -0.7623,  ..., -0.9386,  0.0866, -0.6556],\n",
            "        [-1.3897,  0.3294, -0.7604,  ..., -0.9275,  0.0851, -0.6453],\n",
            "        [-1.4004,  0.3436, -0.7616,  ..., -0.9371,  0.0822, -0.6465]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.1343e-01,  1.3819e-01, -1.5046e-02, -2.6979e-01, -6.1648e-02,\n",
            "          8.5314e-02, -7.7086e-02,  1.3795e-01,  1.5817e-01,  3.7341e-01,\n",
            "          4.1119e-02,  1.1279e-03,  1.9884e+00,  1.3446e-01,  1.9619e+00,\n",
            "          1.1816e-01,  8.4500e-02,  1.0059e+00,  1.3088e+00,  0.0000e+00,\n",
            "          2.1835e+00,  3.7686e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9552, 0.0000,  ..., 0.0000, 2.4045, 0.1483],\n",
            "        [0.0000, 1.4503, 0.1100,  ..., 0.0000, 1.8399, 0.4429],\n",
            "        [0.0000, 2.3674, 0.0000,  ..., 0.0000, 2.1397, 1.3580],\n",
            "        ...,\n",
            "        [0.0000, 1.6171, 0.0000,  ..., 0.0000, 2.4897, 0.6111],\n",
            "        [0.0000, 2.0827, 0.0000,  ..., 0.0000, 2.1303, 0.4323],\n",
            "        [0.0000, 2.0993, 0.0000,  ..., 0.0000, 2.0512, 0.4130]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4204],\n",
            "        [0.5524],\n",
            "        [0.7983],\n",
            "        ...,\n",
            "        [0.4969],\n",
            "        [0.5090],\n",
            "        [0.5064]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5661],\n",
            "        [0.5661],\n",
            "        [0.5661],\n",
            "        ...,\n",
            "        [0.5661],\n",
            "        [0.5661],\n",
            "        [0.5661]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4101, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4101, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4101], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 3\n",
            "node feats are tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0090,  0.0222, -0.0208,  ..., -0.2557, -0.1921, -0.0361],\n",
            "        [-0.0095,  0.0216, -0.0169,  ..., -0.2671, -0.1825, -0.0454],\n",
            "        [-0.0058,  0.0252, -0.0122,  ..., -0.2587, -0.1847, -0.0478],\n",
            "        ...,\n",
            "        [ 0.0010,  0.0228, -0.0202,  ..., -0.2686, -0.1802, -0.0407],\n",
            "        [-0.0007,  0.0286, -0.0160,  ..., -0.2600, -0.1819, -0.0466],\n",
            "        [-0.0016,  0.0250, -0.0293,  ..., -0.2602, -0.1785, -0.0455]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2512, -0.0672, -0.8717,  ..., -1.0152,  0.0332, -0.7538],\n",
            "        [-1.2385, -0.0705, -0.8676,  ..., -1.0103,  0.0358, -0.7495],\n",
            "        [-1.2484, -0.0691, -0.8701,  ..., -1.0153,  0.0287, -0.7419],\n",
            "        ...,\n",
            "        [-1.2474, -0.0470, -0.8735,  ..., -1.0118,  0.0290, -0.7417],\n",
            "        [-1.2419, -0.0598, -0.8686,  ..., -1.0096,  0.0314, -0.7432],\n",
            "        [-1.2538, -0.0761, -0.8767,  ..., -1.0186,  0.0383, -0.7584]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3835,  0.1133, -0.0207, -0.2623, -0.0581,  0.0784, -0.0595,  0.1901,\n",
            "          0.1548,  0.3042,  0.1097,  0.0021,  1.5721,  0.1682,  1.8299,  0.1008,\n",
            "          0.1409,  1.2010,  1.1087,  0.0000,  2.0352,  0.2265]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 0.7277, 0.5233,  ..., 0.0000, 2.4969, 0.0000],\n",
            "        [0.0000, 2.2583, 0.1070,  ..., 0.0000, 2.0713, 0.1879],\n",
            "        [0.0000, 1.6523, 0.0000,  ..., 0.0000, 2.0278, 0.1168],\n",
            "        ...,\n",
            "        [0.0000, 1.8935, 0.0000,  ..., 0.0000, 1.9341, 0.0000],\n",
            "        [0.0000, 1.7143, 0.3072,  ..., 0.0000, 2.4218, 0.1681],\n",
            "        [0.0000, 0.7747, 0.0000,  ..., 0.0000, 1.7146, 0.3110]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.2697],\n",
            "        [0.6320],\n",
            "        [0.4589],\n",
            "        ...,\n",
            "        [0.4857],\n",
            "        [0.5356],\n",
            "        [0.4765]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5608],\n",
            "        [0.5608],\n",
            "        [0.5608],\n",
            "        ...,\n",
            "        [0.5608],\n",
            "        [0.5608],\n",
            "        [0.5608]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4063, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4063, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4063], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 4\n",
            "node feats are tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0279, -0.0097,  0.0283,  ..., -0.2742, -0.1682, -0.0631],\n",
            "        [-0.0161, -0.0087,  0.0439,  ..., -0.2697, -0.1570, -0.0553],\n",
            "        [-0.0166, -0.0137,  0.0316,  ..., -0.2725, -0.1643, -0.0580],\n",
            "        ...,\n",
            "        [-0.0302, -0.0003,  0.0317,  ..., -0.2689, -0.1575, -0.0607],\n",
            "        [-0.0249, -0.0115,  0.0323,  ..., -0.2631, -0.1632, -0.0632],\n",
            "        [-0.0199, -0.0155,  0.0198,  ..., -0.2691, -0.1627, -0.0667]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4114,  0.2148, -0.7660,  ..., -1.0912,  0.2034, -0.8509],\n",
            "        [-1.4198,  0.2152, -0.7691,  ..., -1.0915,  0.2058, -0.8501],\n",
            "        [-1.4255,  0.2362, -0.7782,  ..., -1.0889,  0.2003, -0.8435],\n",
            "        ...,\n",
            "        [-1.4245,  0.2254, -0.7813,  ..., -1.1022,  0.2056, -0.8533],\n",
            "        [-1.4198,  0.2245, -0.7747,  ..., -1.0911,  0.2102, -0.8550],\n",
            "        [-1.4201,  0.2369, -0.7799,  ..., -1.0951,  0.2001, -0.8402]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 3.8322e-01,  1.5701e-01, -1.5348e-02, -2.7071e-01, -4.9279e-02,\n",
            "          9.4210e-02, -9.3477e-02,  1.7662e-01,  1.6854e-01,  3.4866e-01,\n",
            "          2.5551e-02,  6.9183e-04,  1.7203e+00,  1.4811e-01,  2.0880e+00,\n",
            "          8.8046e-02,  7.6628e-02,  8.8350e-01,  1.3268e+00,  0.0000e+00,\n",
            "          2.1647e+00,  2.3576e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.8408, 0.0000,  ..., 0.0000, 2.0528, 0.3329],\n",
            "        [0.0000, 1.4592, 0.0778,  ..., 0.0000, 1.3776, 0.0000],\n",
            "        [0.0000, 0.5770, 0.6159,  ..., 0.0000, 2.0469, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 0.7965, 0.0207,  ..., 0.0000, 2.1400, 0.3935],\n",
            "        [0.0000, 1.8309, 0.0000,  ..., 0.0000, 2.1536, 0.1636],\n",
            "        [0.0000, 1.5492, 0.0000,  ..., 0.0000, 1.5542, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5616],\n",
            "        [0.4979],\n",
            "        [0.3167],\n",
            "        ...,\n",
            "        [0.5001],\n",
            "        [0.4902],\n",
            "        [0.3813]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5936],\n",
            "        [0.5936],\n",
            "        [0.5936],\n",
            "        ...,\n",
            "        [0.5936],\n",
            "        [0.5936],\n",
            "        [0.5936]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4042, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4042, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4042], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 5\n",
            "node feats are tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0131,  0.0267, -0.0011,  ..., -0.2624, -0.1540, -0.0699],\n",
            "        [-0.0222,  0.0205,  0.0038,  ..., -0.2774, -0.1479, -0.0667],\n",
            "        [-0.0187,  0.0227,  0.0015,  ..., -0.2793, -0.1513, -0.0825],\n",
            "        ...,\n",
            "        [-0.0265,  0.0157, -0.0042,  ..., -0.2779, -0.1456, -0.0758],\n",
            "        [-0.0225,  0.0162, -0.0024,  ..., -0.2718, -0.1537, -0.0790],\n",
            "        [-0.0067,  0.0239,  0.0102,  ..., -0.2736, -0.1574, -0.0739]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3719,  0.0897, -0.7838,  ..., -1.0211,  0.1076, -0.8230],\n",
            "        [-1.3667,  0.0876, -0.7785,  ..., -1.0140,  0.1090, -0.8207],\n",
            "        [-1.3667,  0.0963, -0.7804,  ..., -1.0177,  0.1018, -0.8104],\n",
            "        ...,\n",
            "        [-1.3621,  0.0880, -0.7726,  ..., -1.0144,  0.1077, -0.8154],\n",
            "        [-1.3602,  0.0972, -0.7769,  ..., -1.0136,  0.0980, -0.8066],\n",
            "        [-1.3634,  0.0915, -0.7837,  ..., -1.0137,  0.1033, -0.8206]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4149,  0.1049, -0.0194, -0.2685, -0.0517,  0.0916, -0.0434,  0.1723,\n",
            "          0.1714,  0.3621,  0.1175,  0.0050,  1.8274,  0.1591,  2.1273,  0.0400,\n",
            "          0.2182,  1.0636,  1.0237,  0.0000,  1.8965,  0.3149]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7841, 0.0800,  ..., 0.0000, 1.0333, 0.0000],\n",
            "        [0.0000, 2.2879, 0.0534,  ..., 0.0000, 1.4240, 0.0361],\n",
            "        [0.0000, 1.8834, 0.0000,  ..., 0.0000, 1.8262, 0.3648],\n",
            "        ...,\n",
            "        [0.0000, 2.0786, 0.0000,  ..., 0.0000, 2.2622, 0.5754],\n",
            "        [0.0000, 1.3905, 0.0000,  ..., 0.0000, 1.3691, 0.1307],\n",
            "        [0.0000, 2.3556, 0.0000,  ..., 0.0000, 2.5284, 1.3673]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5511],\n",
            "        [0.5096],\n",
            "        [0.5338],\n",
            "        ...,\n",
            "        [0.5839],\n",
            "        [0.6594],\n",
            "        [0.8934]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5559],\n",
            "        [0.5559],\n",
            "        [0.5559],\n",
            "        ...,\n",
            "        [0.5559],\n",
            "        [0.5559],\n",
            "        [0.5559]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4134, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4134, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4134], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 6\n",
            "node feats are tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0069,  0.0173,  0.0295,  ..., -0.2678, -0.1764, -0.0398],\n",
            "        [-0.0138,  0.0125,  0.0235,  ..., -0.2779, -0.1804, -0.0439],\n",
            "        [-0.0014,  0.0188,  0.0202,  ..., -0.2864, -0.1832, -0.0492],\n",
            "        ...,\n",
            "        [-0.0080,  0.0131,  0.0147,  ..., -0.2701, -0.1825, -0.0521],\n",
            "        [-0.0100,  0.0150,  0.0236,  ..., -0.2650, -0.1873, -0.0421],\n",
            "        [ 0.0012,  0.0047,  0.0158,  ..., -0.2772, -0.1867, -0.0528]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3084,  0.0961, -0.7186,  ..., -1.0023,  0.0972, -0.8233],\n",
            "        [-1.3141,  0.0970, -0.7175,  ..., -1.0046,  0.0983, -0.8222],\n",
            "        [-1.3288,  0.0947, -0.7236,  ..., -1.0151,  0.1029, -0.8306],\n",
            "        ...,\n",
            "        [-1.3118,  0.0966, -0.7173,  ..., -1.0069,  0.0994, -0.8298],\n",
            "        [-1.3071,  0.0974, -0.7125,  ..., -0.9995,  0.0941, -0.8196],\n",
            "        [-1.3122,  0.0715, -0.7176,  ..., -1.0051,  0.1128, -0.8432]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3718,  0.1337, -0.0148, -0.2562, -0.0524,  0.0648, -0.0629,  0.1789,\n",
            "          0.1269,  0.3638,  0.0084,  0.0117,  1.8228,  0.2661,  1.7063,  0.0418,\n",
            "          0.0695,  1.0691,  1.4820,  0.0000,  1.8712,  0.2298]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.2546, 0.0394,  ..., 0.0000, 1.5649, 0.0000],\n",
            "        [0.0000, 2.0338, 0.0000,  ..., 0.0000, 2.2393, 0.4710],\n",
            "        [0.0000, 1.9378, 0.1235,  ..., 0.0000, 1.8213, 0.1533],\n",
            "        ...,\n",
            "        [0.0000, 1.9387, 0.1280,  ..., 0.0000, 1.8237, 0.1566],\n",
            "        [0.0000, 3.0613, 0.0000,  ..., 0.0000, 2.0467, 1.2565],\n",
            "        [0.0000, 1.9022, 0.1543,  ..., 0.0000, 1.8129, 0.1386]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4164],\n",
            "        [0.4718],\n",
            "        [0.4676],\n",
            "        ...,\n",
            "        [0.4676],\n",
            "        [0.8696],\n",
            "        [0.4604]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5871],\n",
            "        [0.5871],\n",
            "        [0.5871],\n",
            "        ...,\n",
            "        [0.5871],\n",
            "        [0.5871],\n",
            "        [0.5871]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4027, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4027, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4027], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 7\n",
            "node feats are tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0033,  0.0399,  0.0061,  ..., -0.2625, -0.1738, -0.0487],\n",
            "        [-0.0043,  0.0309,  0.0135,  ..., -0.2741, -0.1661, -0.0514],\n",
            "        [ 0.0011,  0.0322,  0.0024,  ..., -0.2683, -0.1725, -0.0470],\n",
            "        ...,\n",
            "        [ 0.0025,  0.0367,  0.0094,  ..., -0.2702, -0.1749, -0.0507],\n",
            "        [ 0.0028,  0.0379,  0.0111,  ..., -0.2656, -0.1701, -0.0475],\n",
            "        [-0.0045,  0.0364,  0.0122,  ..., -0.2563, -0.1712, -0.0410]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1757,  0.2054, -0.6534,  ..., -0.8608, -0.0688, -0.5401],\n",
            "        [-1.1793,  0.2089, -0.6546,  ..., -0.8614, -0.0745, -0.5363],\n",
            "        [-1.1751,  0.1990, -0.6490,  ..., -0.8620, -0.0710, -0.5349],\n",
            "        ...,\n",
            "        [-1.1679,  0.2056, -0.6496,  ..., -0.8566, -0.0713, -0.5305],\n",
            "        [-1.1746,  0.2054, -0.6487,  ..., -0.8615, -0.0724, -0.5311],\n",
            "        [-1.1779,  0.2011, -0.6540,  ..., -0.8612, -0.0689, -0.5388]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4066,  0.1094, -0.0203, -0.2653, -0.0587,  0.0807, -0.0157,  0.1755,\n",
            "          0.1417,  0.3537,  0.1438,  0.0035,  1.8212,  0.3388,  1.8702,  0.0784,\n",
            "          0.1772,  1.0775,  1.0211,  0.0000,  1.7541,  0.3561]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000e+00, 2.3627e+00, 2.2955e-01,  ..., 0.0000e+00, 1.4034e+00,\n",
            "         7.4343e-01],\n",
            "        [0.0000e+00, 2.0360e+00, 1.3305e-01,  ..., 0.0000e+00, 1.7218e+00,\n",
            "         4.7780e-01],\n",
            "        [0.0000e+00, 2.0294e+00, 2.1175e-03,  ..., 0.0000e+00, 1.0814e+00,\n",
            "         1.3790e-01],\n",
            "        ...,\n",
            "        [0.0000e+00, 1.8989e+00, 2.8228e-01,  ..., 0.0000e+00, 1.7319e+00,\n",
            "         3.1188e-01],\n",
            "        [0.0000e+00, 4.7897e-01, 0.0000e+00,  ..., 0.0000e+00, 1.7475e+00,\n",
            "         7.5113e-01],\n",
            "        [0.0000e+00, 1.8466e+00, 0.0000e+00,  ..., 0.0000e+00, 2.0017e+00,\n",
            "         0.0000e+00]], device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.8126],\n",
            "        [0.6369],\n",
            "        [0.5333],\n",
            "        ...,\n",
            "        [0.5742],\n",
            "        [0.5747],\n",
            "        [0.4966]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5599],\n",
            "        [0.5599],\n",
            "        [0.5599],\n",
            "        ...,\n",
            "        [0.5599],\n",
            "        [0.5599],\n",
            "        [0.5599]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4130, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4130, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4130], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 8\n",
            "node feats are tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0170,  0.0223,  0.0145,  ..., -0.2753, -0.1906, -0.0434],\n",
            "        [-0.0165,  0.0151,  0.0093,  ..., -0.2633, -0.1855, -0.0509],\n",
            "        [-0.0165,  0.0195,  0.0135,  ..., -0.2797, -0.1852, -0.0526],\n",
            "        ...,\n",
            "        [-0.0129,  0.0124,  0.0224,  ..., -0.2721, -0.1869, -0.0376],\n",
            "        [-0.0258,  0.0143,  0.0058,  ..., -0.2699, -0.1778, -0.0440],\n",
            "        [-0.0185,  0.0180,  0.0136,  ..., -0.2763, -0.1832, -0.0451]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3516,  0.2680, -0.7769,  ..., -1.0037, -0.0563, -0.7841],\n",
            "        [-1.3489,  0.2596, -0.7753,  ..., -1.0016, -0.0514, -0.7865],\n",
            "        [-1.3582,  0.2741, -0.7830,  ..., -1.0076, -0.0506, -0.7946],\n",
            "        ...,\n",
            "        [-1.3532,  0.2600, -0.7782,  ..., -1.0007, -0.0534, -0.7916],\n",
            "        [-1.3543,  0.2774, -0.7843,  ..., -1.0072, -0.0567, -0.7842],\n",
            "        [-1.3483,  0.2642, -0.7745,  ..., -1.0006, -0.0556, -0.7844]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3780,  0.1129, -0.0166, -0.2552, -0.0495,  0.0768, -0.0501,  0.1736,\n",
            "          0.1428,  0.3875,  0.0572,  0.0094,  1.7988,  0.1585,  1.9567,  0.0816,\n",
            "          0.0582,  1.0825,  1.3573,  0.0000,  1.7379,  0.3717]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9497, 0.0000,  ..., 0.0000, 1.7013, 0.2878],\n",
            "        [0.0000, 1.5554, 0.0000,  ..., 0.0000, 1.4257, 0.1441],\n",
            "        [0.6060, 0.3329, 0.9212,  ..., 0.0000, 1.7674, 0.0311],\n",
            "        ...,\n",
            "        [0.0000, 1.9252, 0.0000,  ..., 0.0000, 1.6734, 0.4945],\n",
            "        [0.0000, 1.5228, 0.0000,  ..., 0.0000, 1.6238, 0.1997],\n",
            "        [0.0000, 1.9114, 0.0000,  ..., 0.0000, 1.7333, 0.3401]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5268],\n",
            "        [0.4636],\n",
            "        [0.4050],\n",
            "        ...,\n",
            "        [0.5941],\n",
            "        [0.4420],\n",
            "        [0.5329]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5475],\n",
            "        [0.5475],\n",
            "        [0.5475],\n",
            "        ...,\n",
            "        [0.5475],\n",
            "        [0.5475],\n",
            "        [0.5475]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4093, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4093, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4093], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 9\n",
            "node feats are tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0019,  0.0053,  0.0037,  ..., -0.2811, -0.1879, -0.0580],\n",
            "        [ 0.0045,  0.0143,  0.0006,  ..., -0.2800, -0.1834, -0.0528],\n",
            "        [ 0.0006,  0.0165, -0.0061,  ..., -0.2891, -0.1841, -0.0564],\n",
            "        ...,\n",
            "        [-0.0004,  0.0121,  0.0101,  ..., -0.2819, -0.1761, -0.0554],\n",
            "        [-0.0016,  0.0142,  0.0083,  ..., -0.2794, -0.1825, -0.0550],\n",
            "        [-0.0037,  0.0126,  0.0014,  ..., -0.2812, -0.1823, -0.0551]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4123,  0.3127, -0.8576,  ..., -0.9832, -0.0534, -0.8276],\n",
            "        [-1.4007,  0.3049, -0.8510,  ..., -0.9805, -0.0555, -0.8214],\n",
            "        [-1.4042,  0.3017, -0.8571,  ..., -0.9819, -0.0510, -0.8273],\n",
            "        ...,\n",
            "        [-1.4050,  0.3113, -0.8597,  ..., -0.9836, -0.0554, -0.8212],\n",
            "        [-1.4032,  0.3033, -0.8532,  ..., -0.9824, -0.0523, -0.8312],\n",
            "        [-1.4084,  0.3225, -0.8640,  ..., -0.9842, -0.0567, -0.8216]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3840,  0.1212, -0.0148, -0.2580, -0.0494,  0.0771, -0.0608,  0.1653,\n",
            "          0.1391,  0.3967,  0.0160,  0.0095,  1.9360,  0.1734,  1.9400,  0.0881,\n",
            "          0.0438,  1.0444,  1.4437,  0.0000,  1.8083,  0.3169]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.4981, 1.0145,  ..., 0.0000, 2.3483, 0.1497],\n",
            "        [0.0000, 1.2932, 0.0000,  ..., 0.0000, 1.5940, 0.2256],\n",
            "        [0.0000, 2.1293, 0.0000,  ..., 0.0000, 1.7841, 0.2601],\n",
            "        ...,\n",
            "        [0.0000, 2.0189, 0.8951,  ..., 0.0000, 1.9375, 1.0006],\n",
            "        [0.0000, 2.0736, 0.0000,  ..., 0.0000, 1.7076, 0.3012],\n",
            "        [0.0000, 3.6070, 0.0000,  ..., 0.0000, 1.8577, 0.9730]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.3190],\n",
            "        [0.5207],\n",
            "        [0.5235],\n",
            "        ...,\n",
            "        [0.6200],\n",
            "        [0.5382],\n",
            "        [0.8433]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5656],\n",
            "        [0.5656],\n",
            "        [0.5656],\n",
            "        ...,\n",
            "        [0.5656],\n",
            "        [0.5656],\n",
            "        [0.5656]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4090, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4090, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4090], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 10\n",
            "node feats are tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0034, -0.0087, -0.0026,  ..., -0.2535, -0.1882, -0.0545],\n",
            "        [-0.0080, -0.0061, -0.0099,  ..., -0.2565, -0.1777, -0.0576],\n",
            "        [-0.0007, -0.0083, -0.0013,  ..., -0.2520, -0.1870, -0.0496],\n",
            "        ...,\n",
            "        [-0.0014, -0.0007, -0.0059,  ..., -0.2565, -0.1790, -0.0496],\n",
            "        [-0.0048, -0.0030, -0.0071,  ..., -0.2665, -0.1692, -0.0480],\n",
            "        [-0.0052,  0.0009, -0.0037,  ..., -0.2542, -0.1708, -0.0598]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1647,  0.0691, -0.6802,  ..., -0.7894, -0.0851, -0.7714],\n",
            "        [-1.1645,  0.0771, -0.6796,  ..., -0.7909, -0.0809, -0.7713],\n",
            "        [-1.1644,  0.0699, -0.6831,  ..., -0.7920, -0.0866, -0.7751],\n",
            "        ...,\n",
            "        [-1.1611,  0.0679, -0.6789,  ..., -0.7880, -0.0806, -0.7699],\n",
            "        [-1.1623,  0.0670, -0.6831,  ..., -0.7898, -0.0835, -0.7832],\n",
            "        [-1.1590,  0.0750, -0.6821,  ..., -0.7895, -0.0832, -0.7725]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3926,  0.0968, -0.0188, -0.2661, -0.0507,  0.0798, -0.0571,  0.1886,\n",
            "          0.1492,  0.3683,  0.0548,  0.0025,  1.8453,  0.1802,  2.0309,  0.0862,\n",
            "          0.0757,  1.2396,  1.2166,  0.0000,  1.8861,  0.1775]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1313, 0.0000,  ..., 0.0000, 1.8900, 0.3596],\n",
            "        [0.0000, 2.1416, 0.1663,  ..., 0.0000, 1.8369, 0.0000],\n",
            "        [0.0000, 1.9464, 0.0000,  ..., 0.0000, 2.1954, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.9183, 0.0000,  ..., 0.0000, 1.7927, 0.1278],\n",
            "        [0.0000, 1.9168, 0.0000,  ..., 0.0000, 1.7794, 0.0000],\n",
            "        [0.0000, 2.0120, 0.0000,  ..., 0.0000, 1.8355, 0.0544]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5264],\n",
            "        [0.4571],\n",
            "        [0.4384],\n",
            "        ...,\n",
            "        [0.5102],\n",
            "        [0.4709],\n",
            "        [0.4942]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5633],\n",
            "        [0.5633],\n",
            "        [0.5633],\n",
            "        ...,\n",
            "        [0.5633],\n",
            "        [0.5633],\n",
            "        [0.5633]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4082, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4082, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4082], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 11\n",
            "node feats are tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0024,  0.0049,  0.0177,  ..., -0.2654, -0.1818, -0.0971],\n",
            "        [ 0.0029,  0.0199,  0.0151,  ..., -0.2578, -0.1903, -0.0842],\n",
            "        [-0.0022,  0.0125,  0.0214,  ..., -0.2701, -0.1881, -0.0866],\n",
            "        ...,\n",
            "        [-0.0103,  0.0084,  0.0225,  ..., -0.2565, -0.1906, -0.0743],\n",
            "        [-0.0072,  0.0057,  0.0225,  ..., -0.2597, -0.1926, -0.0840],\n",
            "        [-0.0054,  0.0081,  0.0148,  ..., -0.2656, -0.1950, -0.0866]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2497,  0.2399, -0.6286,  ..., -0.7731,  0.0582, -0.6351],\n",
            "        [-1.2537,  0.2328, -0.6326,  ..., -0.7757,  0.0594, -0.6498],\n",
            "        [-1.2554,  0.2496, -0.6375,  ..., -0.7763,  0.0566, -0.6400],\n",
            "        ...,\n",
            "        [-1.2553,  0.2419, -0.6328,  ..., -0.7755,  0.0564, -0.6381],\n",
            "        [-1.2643,  0.2503, -0.6391,  ..., -0.7807,  0.0502, -0.6380],\n",
            "        [-1.2505,  0.2351, -0.6318,  ..., -0.7733,  0.0559, -0.6385]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3750,  0.1267, -0.0154, -0.2592, -0.0496,  0.0704, -0.0661,  0.1927,\n",
            "          0.1294,  0.3675,  0.0144,  0.0164,  1.8571,  0.2860,  1.8356,  0.0738,\n",
            "          0.0801,  1.1285,  1.4629,  0.0000,  1.8798,  0.1466]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.6911, 0.0000,  ..., 0.0000, 1.9051, 0.0000],\n",
            "        [0.0000, 2.5719, 0.0000,  ..., 0.0000, 2.1782, 0.4660],\n",
            "        [1.5907, 0.5053, 1.4188,  ..., 0.0000, 1.1942, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.7808, 0.1438,  ..., 0.0000, 1.5911, 0.0099],\n",
            "        [0.0000, 1.3600, 0.8932,  ..., 0.0000, 2.6640, 0.0000],\n",
            "        [0.0000, 2.3362, 0.0000,  ..., 0.0000, 1.6385, 0.7629]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4324],\n",
            "        [0.6952],\n",
            "        [0.6385],\n",
            "        ...,\n",
            "        [0.4799],\n",
            "        [0.2497],\n",
            "        [0.8138]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5970],\n",
            "        [0.5970],\n",
            "        [0.5970],\n",
            "        ...,\n",
            "        [0.5970],\n",
            "        [0.5970],\n",
            "        [0.5970]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4022, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4022, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4022], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 12\n",
            "node feats are tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-5.6309e-03, -7.3919e-03,  2.0475e-02,  ..., -2.5610e-01,\n",
            "         -1.6503e-01, -4.2019e-02],\n",
            "        [ 5.0623e-03, -6.8299e-03,  2.2305e-02,  ..., -2.6075e-01,\n",
            "         -1.7048e-01, -4.0770e-02],\n",
            "        [-1.6096e-04,  2.6838e-03,  2.5057e-02,  ..., -2.6580e-01,\n",
            "         -1.6899e-01, -3.7826e-02],\n",
            "        ...,\n",
            "        [ 1.2468e-03, -4.3916e-03,  1.5584e-02,  ..., -2.6555e-01,\n",
            "         -1.6782e-01, -4.2439e-02],\n",
            "        [ 2.9763e-03, -8.8665e-03,  1.4279e-02,  ..., -2.7499e-01,\n",
            "         -1.5936e-01, -3.5671e-02],\n",
            "        [ 3.0573e-04, -1.4422e-03,  1.9888e-02,  ..., -2.5536e-01,\n",
            "         -1.7308e-01, -4.1480e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3747,  0.1769, -0.8633,  ..., -1.1571,  0.0636, -0.9190],\n",
            "        [-1.3768,  0.1763, -0.8648,  ..., -1.1564,  0.0597, -0.9117],\n",
            "        [-1.3795,  0.1826, -0.8656,  ..., -1.1582,  0.0608, -0.9144],\n",
            "        ...,\n",
            "        [-1.3785,  0.1611, -0.8607,  ..., -1.1608,  0.0712, -0.9293],\n",
            "        [-1.3728,  0.1790, -0.8615,  ..., -1.1546,  0.0622, -0.9134],\n",
            "        [-1.3873,  0.1696, -0.8684,  ..., -1.1636,  0.0680, -0.9284]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3691,  0.1428, -0.0137, -0.2569, -0.0456,  0.0794, -0.0895,  0.1832,\n",
            "          0.1475,  0.3817, -0.0202,  0.0030,  1.8067,  0.1260,  2.0150,  0.0758,\n",
            "          0.0558,  1.0254,  1.5753,  0.0000,  1.9816,  0.2328]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9682, 0.0000,  ..., 0.0000, 1.9348, 0.3400],\n",
            "        [0.0000, 1.9344, 0.2583,  ..., 0.0000, 1.0051, 0.0000],\n",
            "        [0.0000, 2.0023, 0.0000,  ..., 0.0000, 1.6939, 0.1031],\n",
            "        ...,\n",
            "        [0.0000, 1.8227, 0.0000,  ..., 0.0000, 2.1407, 0.0000],\n",
            "        [0.0000, 1.9000, 0.0000,  ..., 0.0000, 1.8902, 0.2502],\n",
            "        [0.0000, 2.8677, 0.0000,  ..., 0.0000, 2.0877, 1.2257]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5521],\n",
            "        [0.5695],\n",
            "        [0.4247],\n",
            "        ...,\n",
            "        [0.4036],\n",
            "        [0.4590],\n",
            "        [0.8402]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5849],\n",
            "        [0.5849],\n",
            "        [0.5849],\n",
            "        ...,\n",
            "        [0.5849],\n",
            "        [0.5849],\n",
            "        [0.5849]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4030, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4030, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4030], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 13\n",
            "node feats are tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0153,  0.0066,  0.0242,  ..., -0.2773, -0.1769, -0.0365],\n",
            "        [-0.0022,  0.0103,  0.0304,  ..., -0.2838, -0.1755, -0.0468],\n",
            "        [-0.0060,  0.0148,  0.0293,  ..., -0.2824, -0.1847, -0.0496],\n",
            "        ...,\n",
            "        [-0.0122,  0.0038,  0.0290,  ..., -0.2758, -0.1826, -0.0483],\n",
            "        [-0.0207,  0.0160,  0.0295,  ..., -0.2861, -0.1779, -0.0471],\n",
            "        [-0.0124,  0.0092,  0.0303,  ..., -0.2845, -0.1763, -0.0452]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2783,  0.2182, -0.7835,  ..., -0.9942, -0.1708, -0.8425],\n",
            "        [-1.2752,  0.2431, -0.7876,  ..., -0.9948, -0.1797, -0.8221],\n",
            "        [-1.2769,  0.2314, -0.7876,  ..., -1.0006, -0.1782, -0.8319],\n",
            "        ...,\n",
            "        [-1.2749,  0.2265, -0.7865,  ..., -0.9927, -0.1745, -0.8293],\n",
            "        [-1.2723,  0.2398, -0.7845,  ..., -0.9894, -0.1831, -0.8243],\n",
            "        [-1.2770,  0.2322, -0.7894,  ..., -0.9958, -0.1812, -0.8267]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3737,  0.1104, -0.0158, -0.2560, -0.0450,  0.0771, -0.0576,  0.1769,\n",
            "          0.1398,  0.3940,  0.0133,  0.0057,  1.8646,  0.1442,  1.9677,  0.0763,\n",
            "          0.0372,  1.0709,  1.4138,  0.0000,  1.7033,  0.2580]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.0855, 0.0000,  ..., 0.0000, 2.0834, 0.4437],\n",
            "        [0.0000, 1.5052, 0.0000,  ..., 0.0000, 1.2497, 0.0000],\n",
            "        [0.0000, 2.1463, 0.0000,  ..., 0.0000, 1.3301, 0.1179],\n",
            "        ...,\n",
            "        [0.0000, 2.0661, 0.0000,  ..., 0.0000, 1.3211, 0.0000],\n",
            "        [0.0000, 1.1691, 0.1346,  ..., 0.0000, 1.0937, 0.2656],\n",
            "        [0.3001, 0.6717, 1.0390,  ..., 0.0000, 1.7631, 0.2164]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5074],\n",
            "        [0.4830],\n",
            "        [0.4987],\n",
            "        ...,\n",
            "        [0.5738],\n",
            "        [0.5419],\n",
            "        [0.3929]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5619],\n",
            "        [0.5619],\n",
            "        [0.5619],\n",
            "        ...,\n",
            "        [0.5619],\n",
            "        [0.5619],\n",
            "        [0.5619]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4073, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4073, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4073], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 14\n",
            "node feats are tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0224,  0.0028,  0.0024,  ..., -0.2713, -0.1973, -0.0822],\n",
            "        [-0.0310, -0.0031,  0.0010,  ..., -0.2714, -0.1901, -0.0797],\n",
            "        [-0.0280,  0.0016,  0.0012,  ..., -0.2715, -0.1961, -0.0711],\n",
            "        ...,\n",
            "        [-0.0271, -0.0082, -0.0072,  ..., -0.2662, -0.1976, -0.0741],\n",
            "        [-0.0187, -0.0170,  0.0065,  ..., -0.2670, -0.1896, -0.0697],\n",
            "        [-0.0048, -0.0014,  0.0043,  ..., -0.2731, -0.1897, -0.0734]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3474,  0.1216, -0.6365,  ..., -0.9379,  0.0092, -0.7970],\n",
            "        [-1.3395,  0.1198, -0.6290,  ..., -0.9259,  0.0148, -0.7982],\n",
            "        [-1.3432,  0.1223, -0.6334,  ..., -0.9324,  0.0110, -0.7940],\n",
            "        ...,\n",
            "        [-1.3439,  0.1154, -0.6336,  ..., -0.9318,  0.0124, -0.7960],\n",
            "        [-1.3495,  0.1183, -0.6328,  ..., -0.9351,  0.0129, -0.7902],\n",
            "        [-1.3391,  0.1146, -0.6276,  ..., -0.9256,  0.0140, -0.7878]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3500,  0.1232, -0.0168, -0.2506, -0.0484,  0.0651, -0.0626,  0.2211,\n",
            "          0.1260,  0.3547,  0.0607,  0.0181,  1.5987,  0.2970,  1.8651,  0.1070,\n",
            "          0.0550,  1.2583,  1.4968,  0.0000,  1.8207,  0.2099]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.0465, 0.4828,  ..., 0.0000, 1.4784, 0.1847],\n",
            "        [0.0000, 1.1006, 1.3078,  ..., 0.0000, 2.3092, 0.0000],\n",
            "        [0.0000, 1.5645, 0.0300,  ..., 0.0000, 1.5426, 0.2254],\n",
            "        ...,\n",
            "        [0.0000, 1.6348, 0.0000,  ..., 0.0000, 2.1838, 0.3094],\n",
            "        [0.0000, 1.0336, 0.3467,  ..., 0.0000, 1.4240, 0.2739],\n",
            "        [0.0000, 1.0291, 0.9402,  ..., 0.0000, 2.6530, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4585],\n",
            "        [0.2577],\n",
            "        [0.5041],\n",
            "        ...,\n",
            "        [0.4300],\n",
            "        [0.4839],\n",
            "        [0.2141]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5778],\n",
            "        [0.5778],\n",
            "        [0.5778],\n",
            "        ...,\n",
            "        [0.5778],\n",
            "        [0.5778],\n",
            "        [0.5778]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4004, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4004, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4004], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 15\n",
            "node feats are tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0320,  0.0130,  0.0053,  ..., -0.2641, -0.1679, -0.0695],\n",
            "        [ 0.0097,  0.0174,  0.0185,  ..., -0.2749, -0.1587, -0.0757],\n",
            "        [ 0.0180,  0.0116,  0.0013,  ..., -0.2620, -0.1673, -0.0836],\n",
            "        ...,\n",
            "        [ 0.0178,  0.0102,  0.0038,  ..., -0.2605, -0.1592, -0.0732],\n",
            "        [ 0.0228,  0.0124,  0.0017,  ..., -0.2510, -0.1617, -0.0799],\n",
            "        [ 0.0207,  0.0029,  0.0048,  ..., -0.2558, -0.1643, -0.0794]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4143,  0.3045, -0.7606,  ..., -0.8935,  0.0231, -0.7875],\n",
            "        [-1.4047,  0.2855, -0.7506,  ..., -0.8923,  0.0334, -0.8031],\n",
            "        [-1.4079,  0.2897, -0.7589,  ..., -0.8927,  0.0361, -0.8025],\n",
            "        ...,\n",
            "        [-1.4086,  0.2852, -0.7531,  ..., -0.8939,  0.0366, -0.7985],\n",
            "        [-1.4116,  0.2862, -0.7526,  ..., -0.8910,  0.0286, -0.7997],\n",
            "        [-1.4096,  0.2781, -0.7532,  ..., -0.8900,  0.0336, -0.7989]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3997,  0.1191, -0.0155, -0.2625, -0.0540,  0.0789, -0.0604,  0.1645,\n",
            "          0.1443,  0.4005,  0.0482,  0.0078,  1.9791,  0.1983,  2.0402,  0.1093,\n",
            "          0.0546,  1.1306,  1.3935,  0.0000,  1.9310,  0.3772]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.1310, 0.0000,  ..., 0.0000, 1.8503, 0.4181],\n",
            "        [0.0000, 2.3397, 0.0000,  ..., 0.0000, 2.1907, 0.0278],\n",
            "        [0.0000, 2.1187, 0.0000,  ..., 0.0000, 1.8708, 0.3413],\n",
            "        ...,\n",
            "        [0.0000, 2.2218, 0.0000,  ..., 0.0000, 2.2816, 0.8645],\n",
            "        [0.0000, 2.6174, 0.1758,  ..., 0.0000, 1.9762, 0.2249],\n",
            "        [0.0000, 1.1875, 0.1070,  ..., 0.0000, 1.2777, 0.3684]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5563],\n",
            "        [0.5527],\n",
            "        [0.5392],\n",
            "        ...,\n",
            "        [0.6189],\n",
            "        [0.6102],\n",
            "        [0.5464]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5587],\n",
            "        [0.5587],\n",
            "        [0.5587],\n",
            "        ...,\n",
            "        [0.5587],\n",
            "        [0.5587],\n",
            "        [0.5587]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4099, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4099, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4099], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 16\n",
            "node feats are tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0078, -0.0075,  0.0046,  ..., -0.2638, -0.1739, -0.0383],\n",
            "        [ 0.0029, -0.0057,  0.0137,  ..., -0.2738, -0.1772, -0.0421],\n",
            "        [ 0.0020, -0.0062,  0.0056,  ..., -0.2694, -0.1809, -0.0376],\n",
            "        ...,\n",
            "        [-0.0083, -0.0142,  0.0128,  ..., -0.2622, -0.1934, -0.0380],\n",
            "        [ 0.0083, -0.0094,  0.0191,  ..., -0.2715, -0.1764, -0.0427],\n",
            "        [ 0.0118, -0.0110,  0.0215,  ..., -0.2707, -0.1798, -0.0372]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3847,  0.2262, -0.8522,  ..., -1.1193, -0.0116, -0.9535],\n",
            "        [-1.3819,  0.2226, -0.8520,  ..., -1.1258, -0.0147, -0.9456],\n",
            "        [-1.3917,  0.2281, -0.8557,  ..., -1.1317, -0.0117, -0.9520],\n",
            "        ...,\n",
            "        [-1.3787,  0.2150, -0.8503,  ..., -1.1237, -0.0093, -0.9482],\n",
            "        [-1.3830,  0.2319, -0.8548,  ..., -1.1227, -0.0171, -0.9442],\n",
            "        [-1.3793,  0.2194, -0.8481,  ..., -1.1225, -0.0158, -0.9427]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3390,  0.1385, -0.0114, -0.2439, -0.0458,  0.0591, -0.0923,  0.1917,\n",
            "          0.1206,  0.3649, -0.0663,  0.0053,  1.7299,  0.1415,  1.6856,  0.0886,\n",
            "          0.0096,  1.1155,  1.7546,  0.0000,  1.8724,  0.1215]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.2527, 0.0000,  ..., 0.0000, 1.4207, 0.0000],\n",
            "        [0.0000, 2.2939, 0.0000,  ..., 0.0000, 1.3219, 0.3219],\n",
            "        [0.0000, 1.8764, 0.0000,  ..., 0.0000, 1.8298, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.7223, 0.0000,  ..., 0.0000, 1.6417, 0.0000],\n",
            "        [0.0000, 1.9675, 0.0000,  ..., 0.0000, 1.5167, 0.2408],\n",
            "        [0.0000, 0.9517, 0.0000,  ..., 0.0000, 1.4390, 0.2306]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.3858],\n",
            "        [0.5552],\n",
            "        [0.3938],\n",
            "        ...,\n",
            "        [0.3901],\n",
            "        [0.5193],\n",
            "        [0.4335]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5933],\n",
            "        [0.5933],\n",
            "        [0.5933],\n",
            "        ...,\n",
            "        [0.5933],\n",
            "        [0.5933],\n",
            "        [0.5933]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.3964, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.3964, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.3964], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 17\n",
            "node feats are tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0035,  0.0220,  0.0163,  ..., -0.2909, -0.1967, -0.0518],\n",
            "        [ 0.0129,  0.0149,  0.0204,  ..., -0.2859, -0.1948, -0.0594],\n",
            "        [ 0.0126,  0.0064,  0.0132,  ..., -0.2857, -0.2091, -0.0484],\n",
            "        ...,\n",
            "        [ 0.0013,  0.0160,  0.0181,  ..., -0.2915, -0.2000, -0.0553],\n",
            "        [ 0.0005,  0.0174,  0.0194,  ..., -0.2825, -0.1966, -0.0542],\n",
            "        [ 0.0052,  0.0141,  0.0203,  ..., -0.2861, -0.1976, -0.0530]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3324,  0.1930, -0.8706,  ..., -0.9564,  0.0503, -0.7609],\n",
            "        [-1.3342,  0.1928, -0.8732,  ..., -0.9605,  0.0601, -0.7678],\n",
            "        [-1.3319,  0.1870, -0.8699,  ..., -0.9571,  0.0569, -0.7693],\n",
            "        ...,\n",
            "        [-1.3307,  0.1799, -0.8711,  ..., -0.9575,  0.0580, -0.7740],\n",
            "        [-1.3245,  0.1855, -0.8672,  ..., -0.9556,  0.0548, -0.7635],\n",
            "        [-1.3253,  0.2006, -0.8715,  ..., -0.9568,  0.0546, -0.7568]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4019,  0.1190, -0.0155, -0.2668, -0.0503,  0.0828, -0.0667,  0.1571,\n",
            "          0.1530,  0.3932,  0.0062,  0.0052,  2.0105,  0.1364,  2.0173,  0.0530,\n",
            "          0.0812,  1.0278,  1.3447,  0.0000,  1.9285,  0.2674]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.1086, 0.8542, 1.2707,  ..., 0.0000, 2.0870, 0.2353],\n",
            "        [0.0000, 2.1158, 0.0000,  ..., 0.0000, 1.5810, 0.4885],\n",
            "        [0.0000, 1.2359, 0.0000,  ..., 0.0000, 1.6285, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.6965, 0.1893,  ..., 0.0000, 1.3496, 0.0000],\n",
            "        [0.0000, 2.6685, 0.3635,  ..., 0.0000, 2.8051, 0.9080],\n",
            "        [0.0000, 1.9480, 0.0000,  ..., 0.0000, 1.9198, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.3721],\n",
            "        [0.5773],\n",
            "        [0.4488],\n",
            "        ...,\n",
            "        [0.4794],\n",
            "        [0.6510],\n",
            "        [0.5436]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5715],\n",
            "        [0.5715],\n",
            "        [0.5715],\n",
            "        ...,\n",
            "        [0.5715],\n",
            "        [0.5715],\n",
            "        [0.5715]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4090, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4090, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4090], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 18\n",
            "node feats are tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 1.4442e-02,  2.1573e-03, -1.2486e-03,  ..., -2.8920e-01,\n",
            "         -1.9371e-01, -5.4470e-02],\n",
            "        [ 1.1185e-02,  4.3225e-03, -6.6675e-03,  ..., -2.8953e-01,\n",
            "         -1.8374e-01, -5.8882e-02],\n",
            "        [ 1.8083e-02,  5.2212e-03, -9.7704e-03,  ..., -2.9751e-01,\n",
            "         -1.8565e-01, -5.0639e-02],\n",
            "        ...,\n",
            "        [ 1.2139e-02,  1.4581e-02, -5.9284e-03,  ..., -2.9586e-01,\n",
            "         -1.8118e-01, -5.3690e-02],\n",
            "        [-2.9169e-03,  1.3262e-04,  2.9771e-03,  ..., -2.8714e-01,\n",
            "         -1.8487e-01, -4.6615e-02],\n",
            "        [ 8.3836e-03,  7.0534e-03, -8.0493e-03,  ..., -2.9700e-01,\n",
            "         -1.8847e-01, -5.2471e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4310,  0.2784, -0.7485,  ..., -0.9550,  0.0460, -0.7019],\n",
            "        [-1.4413,  0.2691, -0.7473,  ..., -0.9563,  0.0475, -0.7115],\n",
            "        [-1.4362,  0.2695, -0.7484,  ..., -0.9541,  0.0453, -0.7070],\n",
            "        ...,\n",
            "        [-1.4504,  0.2791, -0.7587,  ..., -0.9653,  0.0457, -0.7123],\n",
            "        [-1.4341,  0.2745, -0.7474,  ..., -0.9587,  0.0492, -0.7047],\n",
            "        [-1.4387,  0.2800, -0.7503,  ..., -0.9592,  0.0464, -0.7005]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3963,  0.1213, -0.0156, -0.2582, -0.0568,  0.0767, -0.0531,  0.1560,\n",
            "          0.1408,  0.3875,  0.0622,  0.0086,  1.9298,  0.1917,  1.8983,  0.1092,\n",
            "          0.0837,  1.0771,  1.3656,  0.0000,  1.8848,  0.4097]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.6535, 0.3642,  ..., 0.0000, 1.2615, 0.1615],\n",
            "        [0.0000, 0.9666, 0.0000,  ..., 0.0000, 0.7727, 0.0000],\n",
            "        [0.0000, 2.2039, 0.0000,  ..., 0.0000, 1.4181, 0.3242],\n",
            "        ...,\n",
            "        [0.0000, 2.0517, 0.0000,  ..., 0.0000, 1.8080, 0.3646],\n",
            "        [0.0000, 1.5758, 0.0000,  ..., 0.0000, 1.3339, 0.2920],\n",
            "        [0.0000, 2.0524, 0.0000,  ..., 0.0000, 1.8121, 0.3619]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5478],\n",
            "        [0.4446],\n",
            "        [0.5502],\n",
            "        ...,\n",
            "        [0.5371],\n",
            "        [0.5089],\n",
            "        [0.5374]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5545],\n",
            "        [0.5545],\n",
            "        [0.5545],\n",
            "        ...,\n",
            "        [0.5545],\n",
            "        [0.5545],\n",
            "        [0.5545]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4110, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4110, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4110], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 19\n",
            "node feats are tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0074,  0.0381, -0.0225,  ..., -0.2762, -0.1806, -0.0535],\n",
            "        [ 0.0167,  0.0308, -0.0208,  ..., -0.2671, -0.1757, -0.0434],\n",
            "        [ 0.0143,  0.0387, -0.0219,  ..., -0.2811, -0.1875, -0.0410],\n",
            "        ...,\n",
            "        [ 0.0166,  0.0417, -0.0145,  ..., -0.2655, -0.1920, -0.0463],\n",
            "        [ 0.0028,  0.0399, -0.0309,  ..., -0.2678, -0.1887, -0.0611],\n",
            "        [ 0.0107,  0.0357, -0.0322,  ..., -0.2651, -0.1898, -0.0456]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3067,  0.1392, -0.7506,  ..., -0.8267, -0.0507, -0.6416],\n",
            "        [-1.3161,  0.1334, -0.7522,  ..., -0.8289, -0.0455, -0.6548],\n",
            "        [-1.3107,  0.1457, -0.7573,  ..., -0.8270, -0.0489, -0.6486],\n",
            "        ...,\n",
            "        [-1.3149,  0.1414, -0.7571,  ..., -0.8273, -0.0473, -0.6540],\n",
            "        [-1.3131,  0.1545, -0.7567,  ..., -0.8274, -0.0557, -0.6387],\n",
            "        [-1.3220,  0.1447, -0.7610,  ..., -0.8355, -0.0550, -0.6501]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3901,  0.0919, -0.0197, -0.2557, -0.0597,  0.0652, -0.0263,  0.1636,\n",
            "          0.1321,  0.3517,  0.0867,  0.0142,  1.8146,  0.2283,  1.6475,  0.0588,\n",
            "          0.1007,  1.1953,  1.1589,  0.0000,  1.7285,  0.3098]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.8901, 0.0000,  ..., 0.0000, 1.5569, 0.3821],\n",
            "        [0.0000, 2.0473, 0.1222,  ..., 0.0000, 1.6417, 0.0779],\n",
            "        [0.0000, 1.9348, 0.0471,  ..., 0.0000, 1.6628, 0.2489],\n",
            "        ...,\n",
            "        [0.0000, 1.3551, 1.1585,  ..., 0.0000, 2.2214, 0.0669],\n",
            "        [0.0000, 1.9637, 0.0866,  ..., 0.0000, 1.6255, 0.4945],\n",
            "        [0.0000, 2.2330, 0.0000,  ..., 0.0000, 1.3087, 0.0188]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5650],\n",
            "        [0.4679],\n",
            "        [0.5154],\n",
            "        ...,\n",
            "        [0.3196],\n",
            "        [0.5982],\n",
            "        [0.4759]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5441],\n",
            "        [0.5441],\n",
            "        [0.5441],\n",
            "        ...,\n",
            "        [0.5441],\n",
            "        [0.5441],\n",
            "        [0.5441]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4103, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4103, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4103], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "===== Epoch 5, Loss tensor([28.1423], grad_fn=<DivBackward0>) =====\n",
            "mpika evaluate mode!!\n",
            "x before emb is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612],\n",
            "        [ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612],\n",
            "        [ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612],\n",
            "        ...,\n",
            "        [ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612],\n",
            "        [ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612],\n",
            "        [ 0.0013,  0.0472, -0.0097,  ..., -0.2710, -0.1829, -0.0612]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081],\n",
            "        [-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081],\n",
            "        [-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081],\n",
            "        ...,\n",
            "        [-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081],\n",
            "        [-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081],\n",
            "        [-1.3566,  0.0910, -0.7884,  ..., -0.9106, -0.0146, -0.7081]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3963,  0.0927, -0.0189, -0.2699, -0.0521,  0.0732, -0.0454,  0.1552,\n",
            "          0.1463,  0.3767,  0.0067,  0.0000,  1.9362,  0.1400,  1.8521,  0.0000,\n",
            "          0.0000,  1.1107,  1.1505,  0.0000,  1.7581,  0.2211]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211],\n",
            "        [0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211],\n",
            "        [0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211],\n",
            "        ...,\n",
            "        [0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211],\n",
            "        [0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211],\n",
            "        [0.0000, 1.9362, 0.1400,  ..., 0.0000, 1.7582, 0.2211]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_scores are ->  tensor([[0.5162],\n",
            "        [0.5162],\n",
            "        [0.5162],\n",
            "        ...,\n",
            "        [0.5162],\n",
            "        [0.5162],\n",
            "        [0.5162]], device='cuda:0', grad_fn=<SigmoidBackward0>)   torch.Size([3000, 1])  at epoch:  5\n",
            "global_scores are ->  tensor([[0.5488]], device='cuda:0', grad_fn=<SigmoidBackward0>)   torch.Size([1, 1])  at epoch:  5\n",
            "index of dataloader is 0\n",
            "node feats are tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[73, 67, 92,  ..., 67,  7, 71],\n",
            "        [68, 60, 39,  ..., 77, 99, 33],\n",
            "        [35, 41, 46,  ..., 15, 46, 63],\n",
            "        ...,\n",
            "        [14, 78, 87,  ..., 87, 74, 68],\n",
            "        [87,  0, 54,  ..., 16, 75, 51],\n",
            "        [22, 22, 88,  ..., 83, 69, 12]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0173,  0.0234,  0.0137,  ..., -0.2713, -0.1579, -0.0579],\n",
            "        [-0.0136,  0.0184,  0.0082,  ..., -0.2650, -0.1543, -0.0497],\n",
            "        [-0.0202,  0.0267,  0.0024,  ..., -0.2707, -0.1628, -0.0547],\n",
            "        ...,\n",
            "        [-0.0191,  0.0284,  0.0094,  ..., -0.2718, -0.1575, -0.0541],\n",
            "        [-0.0067,  0.0287,  0.0152,  ..., -0.2780, -0.1486, -0.0606],\n",
            "        [-0.0259,  0.0222,  0.0106,  ..., -0.2732, -0.1639, -0.0524]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3418,  0.0713, -0.7746,  ..., -1.0854,  0.1019, -0.8447],\n",
            "        [-1.3445,  0.0947, -0.7842,  ..., -1.0900,  0.0933, -0.8402],\n",
            "        [-1.3454,  0.0887, -0.7839,  ..., -1.0945,  0.0990, -0.8529],\n",
            "        ...,\n",
            "        [-1.3332,  0.0824, -0.7735,  ..., -1.0771,  0.0963, -0.8341],\n",
            "        [-1.3431,  0.0905, -0.7797,  ..., -1.0833,  0.0929, -0.8439],\n",
            "        [-1.3507,  0.0783, -0.7825,  ..., -1.0895,  0.0992, -0.8505]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.4000,  0.1158, -0.0195, -0.2668, -0.0540,  0.0859, -0.0468,  0.1736,\n",
            "          0.1620,  0.3471,  0.1055,  0.0025,  1.7329,  0.1836,  1.9782,  0.0443,\n",
            "          0.1610,  1.0480,  1.0846,  0.0000,  1.9143,  0.3222]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7089, 0.0000,  ..., 0.0000, 2.2206, 0.0000],\n",
            "        [0.0000, 1.8087, 0.0000,  ..., 0.0000, 1.8153, 0.3474],\n",
            "        [0.0000, 1.8330, 0.0000,  ..., 0.0000, 1.8782, 0.2776],\n",
            "        ...,\n",
            "        [0.0000, 1.8326, 0.0000,  ..., 0.0000, 1.8801, 0.2711],\n",
            "        [0.0000, 1.7973, 0.0000,  ..., 0.0000, 1.8528, 0.3903],\n",
            "        [0.0000, 1.6861, 0.0000,  ..., 0.0000, 1.6509, 0.1738]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4368],\n",
            "        [0.5610],\n",
            "        [0.5343],\n",
            "        ...,\n",
            "        [0.5382],\n",
            "        [0.5095],\n",
            "        [0.6021]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5569],\n",
            "        [0.5569],\n",
            "        [0.5569],\n",
            "        ...,\n",
            "        [0.5569],\n",
            "        [0.5569],\n",
            "        [0.5569]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4114, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4114, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4114], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 1\n",
            "node feats are tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[36, 11, 86,  ..., 91, 89,  1],\n",
            "        [ 0, 80,  0,  ..., 21, 68, 49],\n",
            "        [35, 91, 82,  ..., 65,  9, 48],\n",
            "        ...,\n",
            "        [36, 31, 23,  ..., 93, 51, 39],\n",
            "        [33, 62, 61,  ..., 49, 31, 90],\n",
            "        [36, 62, 58,  ..., 69, 33, 21]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0018,  0.0426, -0.0073,  ..., -0.2652, -0.1786, -0.0590],\n",
            "        [ 0.0034,  0.0449, -0.0126,  ..., -0.2613, -0.1712, -0.0542],\n",
            "        [ 0.0054,  0.0538, -0.0114,  ..., -0.2649, -0.1829, -0.0510],\n",
            "        ...,\n",
            "        [ 0.0066,  0.0432, -0.0175,  ..., -0.2630, -0.1746, -0.0535],\n",
            "        [ 0.0091,  0.0522, -0.0227,  ..., -0.2653, -0.1838, -0.0538],\n",
            "        [ 0.0013,  0.0502, -0.0117,  ..., -0.2619, -0.1767, -0.0508]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2853,  0.0939, -0.7617,  ..., -0.8715, -0.0728, -0.6640],\n",
            "        [-1.2757,  0.0981, -0.7551,  ..., -0.8651, -0.0737, -0.6512],\n",
            "        [-1.2789,  0.0894, -0.7560,  ..., -0.8661, -0.0737, -0.6638],\n",
            "        ...,\n",
            "        [-1.2817,  0.0839, -0.7577,  ..., -0.8687, -0.0706, -0.6651],\n",
            "        [-1.2834,  0.0957, -0.7566,  ..., -0.8696, -0.0755, -0.6580],\n",
            "        [-1.2816,  0.1005, -0.7572,  ..., -0.8663, -0.0764, -0.6482]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3895,  0.0973, -0.0209, -0.2594, -0.0622,  0.0656, -0.0236,  0.1610,\n",
            "          0.1333,  0.3255,  0.0963,  0.0078,  1.7533,  0.2429,  1.5612,  0.0592,\n",
            "          0.1095,  1.1443,  1.0564,  0.0000,  1.7613,  0.2757]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9130, 0.0608,  ..., 0.0000, 1.7864, 0.1322],\n",
            "        [0.0000, 2.0005, 0.0000,  ..., 0.0000, 1.8883, 0.1169],\n",
            "        [0.0000, 2.8267, 0.0000,  ..., 0.0000, 1.9671, 1.1718],\n",
            "        ...,\n",
            "        [0.0000, 2.3318, 0.3762,  ..., 0.0000, 1.8143, 0.0744],\n",
            "        [0.0000, 1.4766, 0.4842,  ..., 0.0000, 1.2762, 0.0000],\n",
            "        [0.0000, 1.8640, 0.0944,  ..., 0.0000, 1.7231, 0.2063]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5005],\n",
            "        [0.5028],\n",
            "        [0.8546],\n",
            "        ...,\n",
            "        [0.5756],\n",
            "        [0.4766],\n",
            "        [0.5124]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5484],\n",
            "        [0.5484],\n",
            "        [0.5484],\n",
            "        ...,\n",
            "        [0.5484],\n",
            "        [0.5484],\n",
            "        [0.5484]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4107, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4107, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4107], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 2\n",
            "node feats are tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[71, 52,  3,  ..., 35, 18, 88],\n",
            "        [23, 45, 18,  ..., 75, 40, 76],\n",
            "        [15, 36, 60,  ..., 72, 67,  8],\n",
            "        ...,\n",
            "        [ 3, 86, 98,  ..., 42, 83, 78],\n",
            "        [98,  6, 83,  ..., 45, 65, 11],\n",
            "        [11, 42, 73,  ...,  9, 23, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0007,  0.0362,  0.0064,  ..., -0.2615, -0.1707, -0.0577],\n",
            "        [ 0.0044,  0.0306,  0.0025,  ..., -0.2696, -0.1743, -0.0476],\n",
            "        [ 0.0098,  0.0353, -0.0089,  ..., -0.2697, -0.1810, -0.0525],\n",
            "        ...,\n",
            "        [ 0.0121,  0.0366, -0.0035,  ..., -0.2733, -0.1775, -0.0537],\n",
            "        [ 0.0073,  0.0298, -0.0040,  ..., -0.2637, -0.1800, -0.0507],\n",
            "        [ 0.0101,  0.0291,  0.0072,  ..., -0.2584, -0.1781, -0.0512]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.1346,  0.2157, -0.6325,  ..., -0.7886, -0.1320, -0.5733],\n",
            "        [-1.1351,  0.2371, -0.6362,  ..., -0.7902, -0.1420, -0.5528],\n",
            "        [-1.1448,  0.2035, -0.6354,  ..., -0.7971, -0.1305, -0.5809],\n",
            "        ...,\n",
            "        [-1.1409,  0.2298, -0.6382,  ..., -0.7954, -0.1377, -0.5570],\n",
            "        [-1.1380,  0.2252, -0.6329,  ..., -0.7897, -0.1383, -0.5649],\n",
            "        [-1.1354,  0.2308, -0.6311,  ..., -0.7908, -0.1356, -0.5604]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3904,  0.1061, -0.0200, -0.2606, -0.0573,  0.0749, -0.0202,  0.1878,\n",
            "          0.1326,  0.3599,  0.1349,  0.0062,  1.7704,  0.3414,  1.8626,  0.1155,\n",
            "          0.1090,  1.1654,  1.1247,  0.0000,  1.7076,  0.3503]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9150, 0.0000,  ..., 0.0000, 1.3767, 0.0000],\n",
            "        [0.0000, 1.4315, 0.2960,  ..., 0.0000, 1.3426, 0.1114],\n",
            "        [0.0000, 1.3250, 1.2344,  ..., 0.0000, 2.0956, 0.0676],\n",
            "        ...,\n",
            "        [0.0000, 1.4406, 0.1689,  ..., 0.0000, 1.6042, 0.0000],\n",
            "        [0.0000, 1.8300, 0.1516,  ..., 0.0000, 1.5586, 0.3589],\n",
            "        [0.0000, 1.9679, 0.0000,  ..., 0.0000, 2.0792, 0.5311]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4388],\n",
            "        [0.5463],\n",
            "        [0.3519],\n",
            "        ...,\n",
            "        [0.4267],\n",
            "        [0.5804],\n",
            "        [0.5215]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5538],\n",
            "        [0.5538],\n",
            "        [0.5538],\n",
            "        ...,\n",
            "        [0.5538],\n",
            "        [0.5538],\n",
            "        [0.5538]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4106, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4106, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4106], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 3\n",
            "node feats are tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 98, 84,  ..., 33,  4, 98],\n",
            "        [72, 59, 79,  ..., 94, 24, 11],\n",
            "        [45,  4, 11,  ..., 85,  2, 34],\n",
            "        ...,\n",
            "        [55, 63, 60,  ..., 72,  8, 58],\n",
            "        [73, 72, 40,  ..., 24, 35, 16],\n",
            "        [22, 77, 22,  ..., 30, 59, 43]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 6.1239e-05,  1.4465e-02,  5.7520e-03,  ..., -2.7865e-01,\n",
            "         -1.4922e-01, -3.3237e-02],\n",
            "        [-7.5009e-03,  9.2759e-03, -7.9413e-04,  ..., -2.7555e-01,\n",
            "         -1.6430e-01, -4.0761e-02],\n",
            "        [-1.0448e-02,  1.3261e-02,  8.2156e-03,  ..., -2.7354e-01,\n",
            "         -1.7012e-01, -4.8615e-02],\n",
            "        ...,\n",
            "        [-2.0485e-02,  1.7595e-02,  7.8419e-03,  ..., -2.7751e-01,\n",
            "         -1.6319e-01, -2.7531e-02],\n",
            "        [-1.3432e-02,  1.0122e-02,  4.4131e-03,  ..., -2.7689e-01,\n",
            "         -1.6068e-01, -3.3883e-02],\n",
            "        [-1.6939e-02,  1.7961e-02, -6.1872e-03,  ..., -2.7172e-01,\n",
            "         -1.6140e-01, -3.4825e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3268e+00,  1.6125e-01, -7.6797e-01,  ..., -1.0659e+00,\n",
            "         -3.6508e-03, -8.8859e-01],\n",
            "        [-1.3104e+00,  1.5449e-01, -7.5685e-01,  ..., -1.0497e+00,\n",
            "          2.0238e-03, -8.8298e-01],\n",
            "        [-1.3169e+00,  1.5641e-01, -7.6133e-01,  ..., -1.0533e+00,\n",
            "         -5.3815e-03, -8.8400e-01],\n",
            "        ...,\n",
            "        [-1.3190e+00,  1.5793e-01, -7.6098e-01,  ..., -1.0593e+00,\n",
            "         -7.2413e-03, -8.8080e-01],\n",
            "        [-1.3164e+00,  1.4965e-01, -7.6073e-01,  ..., -1.0571e+00,\n",
            "          2.8016e-04, -8.9556e-01],\n",
            "        [-1.3227e+00,  1.6451e-01, -7.6627e-01,  ..., -1.0615e+00,\n",
            "         -6.3136e-03, -8.7689e-01]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3563,  0.1293, -0.0152, -0.2519, -0.0482,  0.0704, -0.0705,  0.1913,\n",
            "          0.1332,  0.3603,  0.0193,  0.0080,  1.6874,  0.1799,  1.8162,  0.0973,\n",
            "          0.0428,  1.0968,  1.4931,  0.0000,  1.8336,  0.2282]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.8854, 0.0000,  ..., 0.0000, 1.8095, 0.1623],\n",
            "        [0.0000, 1.4507, 1.1194,  ..., 0.0000, 2.3543, 0.0000],\n",
            "        [0.0000, 2.3327, 0.0000,  ..., 0.0000, 1.8733, 0.8877],\n",
            "        ...,\n",
            "        [0.0000, 1.3329, 0.0000,  ..., 0.0000, 1.8510, 0.3451],\n",
            "        [0.0000, 1.5493, 0.0000,  ..., 0.0000, 1.1542, 0.0000],\n",
            "        [0.0000, 1.7508, 0.0000,  ..., 0.0000, 2.4830, 0.2884]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4457],\n",
            "        [0.2536],\n",
            "        [0.7742],\n",
            "        ...,\n",
            "        [0.4232],\n",
            "        [0.4132],\n",
            "        [0.4090]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5736],\n",
            "        [0.5736],\n",
            "        [0.5736],\n",
            "        ...,\n",
            "        [0.5736],\n",
            "        [0.5736],\n",
            "        [0.5736]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4026, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4026, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4026], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 4\n",
            "node feats are tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[67, 77, 61,  ..., 39, 97, 52],\n",
            "        [47, 55, 93,  ..., 75,  8, 25],\n",
            "        [85, 39, 49,  ..., 52, 40, 42],\n",
            "        ...,\n",
            "        [66, 74, 30,  ..., 42, 47, 35],\n",
            "        [83, 96, 97,  ..., 88, 14, 72],\n",
            "        [81, 15, 99,  ..., 17, 47, 46]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0049,  0.0140,  0.0132,  ..., -0.2688, -0.1767, -0.0562],\n",
            "        [ 0.0099,  0.0136,  0.0175,  ..., -0.2847, -0.1921, -0.0705],\n",
            "        [ 0.0095,  0.0135,  0.0131,  ..., -0.2720, -0.1852, -0.0546],\n",
            "        ...,\n",
            "        [ 0.0151,  0.0091,  0.0034,  ..., -0.2722, -0.1906, -0.0618],\n",
            "        [ 0.0121,  0.0071,  0.0043,  ..., -0.2741, -0.1875, -0.0559],\n",
            "        [ 0.0038,  0.0129,  0.0127,  ..., -0.2733, -0.1857, -0.0608]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.5135,  0.2661, -0.8128,  ..., -1.0189,  0.0363, -0.6885],\n",
            "        [-1.5023,  0.2520, -0.8069,  ..., -1.0181,  0.0450, -0.6973],\n",
            "        [-1.5117,  0.2611, -0.8074,  ..., -1.0208,  0.0342, -0.6871],\n",
            "        ...,\n",
            "        [-1.5030,  0.2491, -0.8059,  ..., -1.0159,  0.0454, -0.6980],\n",
            "        [-1.5146,  0.2603, -0.8097,  ..., -1.0243,  0.0369, -0.6855],\n",
            "        [-1.4939,  0.2620, -0.8043,  ..., -1.0120,  0.0430, -0.6860]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.1118e-01,  1.0909e-01, -1.7943e-02, -2.6647e-01, -5.9652e-02,\n",
            "          8.1712e-02, -3.9793e-02,  1.3291e-01,  1.5277e-01,  3.6870e-01,\n",
            "          7.0598e-02,  1.8498e-03,  1.9465e+00,  1.3882e-01,  1.8217e+00,\n",
            "          6.4727e-02,  9.6752e-02,  9.9025e-01,  1.1079e+00,  0.0000e+00,\n",
            "          1.8548e+00,  3.8803e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.5131, 0.0000,  ..., 0.0000, 1.3376, 0.0000],\n",
            "        [0.0000, 2.2342, 0.0997,  ..., 0.0000, 1.5792, 0.0368],\n",
            "        [0.0000, 2.0463, 0.0000,  ..., 0.0000, 1.8168, 0.3426],\n",
            "        ...,\n",
            "        [0.0000, 2.0451, 0.0000,  ..., 0.0000, 1.8103, 0.4906],\n",
            "        [0.0000, 1.3907, 0.0000,  ..., 0.0000, 1.3734, 1.1215],\n",
            "        [0.0000, 1.7010, 0.0000,  ..., 0.0000, 1.5302, 0.1771]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5077],\n",
            "        [0.5911],\n",
            "        [0.5616],\n",
            "        ...,\n",
            "        [0.6185],\n",
            "        [0.7351],\n",
            "        [0.5316]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5433],\n",
            "        [0.5433],\n",
            "        [0.5433],\n",
            "        ...,\n",
            "        [0.5433],\n",
            "        [0.5433],\n",
            "        [0.5433]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4116, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4116, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4116], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 5\n",
            "node feats are tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 48,  3,  ..., 25, 21, 73],\n",
            "        [96, 57, 15,  ..., 72, 30, 92],\n",
            "        [73, 49, 82,  ..., 92, 38, 64],\n",
            "        ...,\n",
            "        [47, 66, 59,  ..., 31, 28, 25],\n",
            "        [66, 78, 18,  ..., 79, 18, 24],\n",
            "        [31, 37, 86,  ..., 46, 49, 27]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0190, -0.0027,  0.0250,  ..., -0.2757, -0.1591, -0.0634],\n",
            "        [-0.0147, -0.0136,  0.0224,  ..., -0.2696, -0.1508, -0.0666],\n",
            "        [-0.0231, -0.0140,  0.0278,  ..., -0.2732, -0.1628, -0.0578],\n",
            "        ...,\n",
            "        [-0.0132, -0.0165,  0.0319,  ..., -0.2817, -0.1525, -0.0564],\n",
            "        [-0.0112, -0.0089,  0.0204,  ..., -0.2841, -0.1597, -0.0541],\n",
            "        [-0.0209, -0.0110,  0.0367,  ..., -0.2864, -0.1554, -0.0636]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4176,  0.2160, -0.7131,  ..., -1.1580,  0.1955, -0.7455],\n",
            "        [-1.4126,  0.2215, -0.7139,  ..., -1.1550,  0.1996, -0.7460],\n",
            "        [-1.4140,  0.2275, -0.7130,  ..., -1.1549,  0.1883, -0.7408],\n",
            "        ...,\n",
            "        [-1.4021,  0.2221, -0.7120,  ..., -1.1506,  0.1973, -0.7432],\n",
            "        [-1.4066,  0.2020, -0.7052,  ..., -1.1493,  0.1940, -0.7518],\n",
            "        [-1.4134,  0.2246, -0.7147,  ..., -1.1584,  0.1944, -0.7470]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 3.7759e-01,  1.5958e-01, -1.5217e-02, -2.6403e-01, -5.2412e-02,\n",
            "          9.1569e-02, -8.6275e-02,  1.7621e-01,  1.6140e-01,  3.4225e-01,\n",
            "          5.9951e-02,  4.8128e-04,  1.6557e+00,  1.6369e-01,  2.0028e+00,\n",
            "          1.2315e-01,  1.0042e-01,  8.8243e-01,  1.3442e+00,  0.0000e+00,\n",
            "          2.1124e+00,  3.1436e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.1114, 0.0000,  ..., 0.0000, 2.2077, 0.5298],\n",
            "        [0.0000, 1.1064, 1.1105,  ..., 0.0000, 2.6345, 0.1298],\n",
            "        [0.0000, 1.5577, 0.0341,  ..., 0.0000, 1.5438, 0.3135],\n",
            "        ...,\n",
            "        [0.0000, 1.4109, 0.2508,  ..., 0.0000, 1.6466, 0.0049],\n",
            "        [0.0000, 1.8696, 1.6645,  ..., 0.0000, 2.8868, 0.0000],\n",
            "        [0.0000, 1.5481, 0.0000,  ..., 0.0000, 1.8996, 0.3932]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5607],\n",
            "        [0.2955],\n",
            "        [0.5450],\n",
            "        ...,\n",
            "        [0.4793],\n",
            "        [0.3764],\n",
            "        [0.5770]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5841],\n",
            "        [0.5841],\n",
            "        [0.5841],\n",
            "        ...,\n",
            "        [0.5841],\n",
            "        [0.5841],\n",
            "        [0.5841]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4058, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4058, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4058], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 6\n",
            "node feats are tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[37,  3, 80,  ..., 91,  8, 89],\n",
            "        [48, 95, 54,  ..., 12, 61, 80],\n",
            "        [25, 92, 65,  ..., 24, 81, 56],\n",
            "        ...,\n",
            "        [77, 21, 88,  ..., 65, 31,  3],\n",
            "        [83, 58, 77,  ..., 95, 83, 67],\n",
            "        [93, 22, 31,  ..., 24, 77, 45]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0071,  0.0230,  0.0126,  ..., -0.2685, -0.1937, -0.0615],\n",
            "        [ 0.0036,  0.0160,  0.0116,  ..., -0.2678, -0.1974, -0.0657],\n",
            "        [-0.0021,  0.0267,  0.0118,  ..., -0.2680, -0.1874, -0.0606],\n",
            "        ...,\n",
            "        [ 0.0017,  0.0222,  0.0065,  ..., -0.2629, -0.1908, -0.0759],\n",
            "        [ 0.0083,  0.0244,  0.0142,  ..., -0.2624, -0.1886, -0.0603],\n",
            "        [-0.0012,  0.0219,  0.0097,  ..., -0.2711, -0.1985, -0.0646]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2545,  0.2052, -0.7072,  ..., -0.8248,  0.0649, -0.6871],\n",
            "        [-1.2535,  0.1996, -0.7059,  ..., -0.8227,  0.0699, -0.6961],\n",
            "        [-1.2572,  0.1893, -0.7060,  ..., -0.8254,  0.0780, -0.7014],\n",
            "        ...,\n",
            "        [-1.2554,  0.1936, -0.7098,  ..., -0.8230,  0.0740, -0.7026],\n",
            "        [-1.2665,  0.2053, -0.7139,  ..., -0.8316,  0.0699, -0.7016],\n",
            "        [-1.2554,  0.2038, -0.7055,  ..., -0.8221,  0.0712, -0.6918]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 3.7922e-01,  1.2812e-01, -1.5063e-02, -2.5983e-01, -4.8202e-02,\n",
            "          7.5467e-02, -7.1812e-02,  1.8292e-01,  1.3890e-01,  3.7527e-01,\n",
            "          9.6352e-04,  8.2346e-03,  1.8777e+00,  2.0735e-01,  1.9054e+00,\n",
            "          6.0333e-02,  8.6714e-02,  1.0735e+00,  1.4629e+00,  0.0000e+00,\n",
            "          1.9033e+00,  1.7890e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.0080, 0.0693,  ..., 0.0000, 1.8738, 0.0665],\n",
            "        [0.0000, 2.4343, 0.2631,  ..., 0.0000, 1.9695, 0.0000],\n",
            "        [0.0000, 1.4940, 0.0000,  ..., 0.0000, 2.1599, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.6697, 0.2337,  ..., 0.0000, 1.3927, 0.0000],\n",
            "        [0.0000, 2.5435, 0.0000,  ..., 0.0000, 1.5881, 0.8378],\n",
            "        [0.0000, 1.8525, 0.0000,  ..., 0.0000, 1.3271, 0.1822]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4682],\n",
            "        [0.5388],\n",
            "        [0.3737],\n",
            "        ...,\n",
            "        [0.4598],\n",
            "        [0.8140],\n",
            "        [0.5028]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5902],\n",
            "        [0.5902],\n",
            "        [0.5902],\n",
            "        ...,\n",
            "        [0.5902],\n",
            "        [0.5902],\n",
            "        [0.5902]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4035, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4035, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4035], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 7\n",
            "node feats are tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[87, 16,  8,  ..., 23, 88, 13],\n",
            "        [95, 86, 45,  ..., 23, 99, 67],\n",
            "        [84, 85, 28,  ..., 87, 25, 42],\n",
            "        ...,\n",
            "        [17, 42, 88,  ..., 22, 11, 64],\n",
            "        [72, 96, 76,  ..., 53, 39, 78],\n",
            "        [13, 66, 26,  ..., 66, 48, 61]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0048,  0.0162,  0.0207,  ..., -0.2697, -0.1898, -0.0547],\n",
            "        [ 0.0007,  0.0059,  0.0094,  ..., -0.2679, -0.1891, -0.0444],\n",
            "        [ 0.0029,  0.0133,  0.0216,  ..., -0.2755, -0.1922, -0.0523],\n",
            "        ...,\n",
            "        [ 0.0041,  0.0163,  0.0111,  ..., -0.2590, -0.1863, -0.0515],\n",
            "        [ 0.0019,  0.0163,  0.0114,  ..., -0.2674, -0.1887, -0.0522],\n",
            "        [ 0.0084,  0.0172,  0.0191,  ..., -0.2693, -0.1956, -0.0500]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3730,  0.1444, -0.9133,  ..., -0.9613,  0.1006, -0.8614],\n",
            "        [-1.3619,  0.1619, -0.9079,  ..., -0.9559,  0.0888, -0.8453],\n",
            "        [-1.3730,  0.1613, -0.9161,  ..., -0.9629,  0.0967, -0.8606],\n",
            "        ...,\n",
            "        [-1.3827,  0.1577, -0.9216,  ..., -0.9686,  0.0921, -0.8623],\n",
            "        [-1.3727,  0.1398, -0.9122,  ..., -0.9602,  0.0993, -0.8673],\n",
            "        [-1.3740,  0.1499, -0.9134,  ..., -0.9602,  0.0992, -0.8623]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.1643e-01,  1.2085e-01, -1.5608e-02, -2.7424e-01, -5.3499e-02,\n",
            "          8.4176e-02, -6.6303e-02,  1.3922e-01,  1.5740e-01,  3.9089e-01,\n",
            "         -7.2228e-03,  1.1498e-03,  2.1074e+00,  1.3547e-01,  1.9759e+00,\n",
            "          2.7092e-02,  7.6910e-02,  9.7404e-01,  1.2706e+00,  0.0000e+00,\n",
            "          2.0024e+00,  2.6672e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 3.1731, 0.0000,  ..., 0.0000, 2.1999, 0.9839],\n",
            "        [0.0000, 2.2498, 0.0000,  ..., 0.0000, 2.0518, 0.0000],\n",
            "        [0.0000, 2.1553, 0.0000,  ..., 0.0000, 1.5520, 0.4499],\n",
            "        ...,\n",
            "        [0.0000, 2.2555, 0.0000,  ..., 0.0000, 1.9681, 0.2139],\n",
            "        [0.0000, 1.8343, 0.0000,  ..., 0.0000, 1.6959, 0.1885],\n",
            "        [0.0000, 2.2564, 0.0000,  ..., 0.0000, 1.9715, 0.2080]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.8173],\n",
            "        [0.4622],\n",
            "        [0.6525],\n",
            "        ...,\n",
            "        [0.5435],\n",
            "        [0.5889],\n",
            "        [0.5421]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5742],\n",
            "        [0.5742],\n",
            "        [0.5742],\n",
            "        ...,\n",
            "        [0.5742],\n",
            "        [0.5742],\n",
            "        [0.5742]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4085, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4085, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4085], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 8\n",
            "node feats are tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[94, 59, 32,  ..., 15, 42, 82],\n",
            "        [65, 14, 97,  ..., 77, 90, 34],\n",
            "        [49, 42, 65,  ..., 19, 85, 29],\n",
            "        ...,\n",
            "        [ 8, 62, 40,  ..., 48, 78, 87],\n",
            "        [73, 61, 56,  ..., 78, 82, 88],\n",
            "        [77, 79, 92,  ..., 30, 77, 63]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0019, -0.0160,  0.0202,  ..., -0.2625, -0.1629, -0.0537],\n",
            "        [ 0.0010, -0.0075,  0.0306,  ..., -0.2589, -0.1549, -0.0371],\n",
            "        [ 0.0061, -0.0184,  0.0202,  ..., -0.2569, -0.1625, -0.0487],\n",
            "        ...,\n",
            "        [-0.0010, -0.0073,  0.0227,  ..., -0.2588, -0.1633, -0.0448],\n",
            "        [ 0.0051, -0.0129,  0.0254,  ..., -0.2537, -0.1594, -0.0361],\n",
            "        [ 0.0152, -0.0175,  0.0275,  ..., -0.2577, -0.1557, -0.0444]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4494,  0.2323, -0.8455,  ..., -1.1579,  0.0591, -0.9074],\n",
            "        [-1.4423,  0.2333, -0.8377,  ..., -1.1483,  0.0562, -0.8983],\n",
            "        [-1.4416,  0.2425, -0.8406,  ..., -1.1480,  0.0586, -0.8905],\n",
            "        ...,\n",
            "        [-1.4486,  0.2227, -0.8444,  ..., -1.1546,  0.0651, -0.9188],\n",
            "        [-1.4386,  0.2144, -0.8337,  ..., -1.1482,  0.0633, -0.9053],\n",
            "        [-1.4475,  0.2297, -0.8430,  ..., -1.1549,  0.0547, -0.9053]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3575,  0.1487, -0.0115, -0.2504, -0.0432,  0.0739, -0.0930,  0.1805,\n",
            "          0.1376,  0.3969, -0.0566,  0.0054,  1.8403,  0.1180,  1.9627,  0.0707,\n",
            "          0.0267,  0.9965,  1.7354,  0.0000,  1.9052,  0.2531]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9774, 0.0000,  ..., 0.0000, 1.8196, 0.2047],\n",
            "        [0.0000, 1.6701, 0.2114,  ..., 0.0000, 1.2748, 0.0000],\n",
            "        [0.0000, 1.3435, 0.8374,  ..., 0.0000, 2.4670, 0.0824],\n",
            "        ...,\n",
            "        [0.6667, 1.3021, 1.6653,  ..., 0.0000, 2.3560, 0.0000],\n",
            "        [0.0000, 1.9752, 0.0000,  ..., 0.0000, 1.8111, 0.1874],\n",
            "        [0.0000, 1.3667, 0.0000,  ..., 0.0000, 1.5853, 0.3518]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.4671],\n",
            "        [0.4155],\n",
            "        [0.2532],\n",
            "        ...,\n",
            "        [0.4295],\n",
            "        [0.4628],\n",
            "        [0.4881]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5867],\n",
            "        [0.5867],\n",
            "        [0.5867],\n",
            "        ...,\n",
            "        [0.5867],\n",
            "        [0.5867],\n",
            "        [0.5867]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4017, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4017, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4017], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 9\n",
            "node feats are tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[77, 85, 42,  ..., 51, 74, 49],\n",
            "        [93, 23, 23,  ..., 77, 96, 83],\n",
            "        [59, 79, 89,  ..., 75, 20, 12],\n",
            "        ...,\n",
            "        [ 6, 68, 58,  ..., 22,  2, 89],\n",
            "        [95, 74, 84,  ..., 58,  8, 29],\n",
            "        [49, 17,  3,  ..., 54, 36, 74]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0185,  0.0159,  0.0157,  ..., -0.2657, -0.1717, -0.0609],\n",
            "        [ 0.0158,  0.0131,  0.0145,  ..., -0.2639, -0.1600, -0.0587],\n",
            "        [ 0.0142,  0.0127,  0.0217,  ..., -0.2695, -0.1598, -0.0580],\n",
            "        ...,\n",
            "        [ 0.0140,  0.0165,  0.0100,  ..., -0.2708, -0.1641, -0.0594],\n",
            "        [ 0.0263,  0.0173,  0.0159,  ..., -0.2779, -0.1656, -0.0641],\n",
            "        [ 0.0187,  0.0157,  0.0142,  ..., -0.2752, -0.1624, -0.0602]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3854,  0.3514, -0.7515,  ..., -0.9303,  0.0306, -0.8708],\n",
            "        [-1.3751,  0.3549, -0.7505,  ..., -0.9332,  0.0347, -0.8654],\n",
            "        [-1.3793,  0.3473, -0.7529,  ..., -0.9314,  0.0315, -0.8743],\n",
            "        ...,\n",
            "        [-1.3841,  0.3522, -0.7526,  ..., -0.9329,  0.0315, -0.8756],\n",
            "        [-1.3857,  0.3561, -0.7565,  ..., -0.9353,  0.0307, -0.8736],\n",
            "        [-1.3772,  0.3434, -0.7517,  ..., -0.9322,  0.0385, -0.8804]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3776,  0.1391, -0.0134, -0.2560, -0.0513,  0.0757, -0.0798,  0.1743,\n",
            "          0.1378,  0.3938,  0.0078,  0.0084,  1.8893,  0.1826,  1.9786,  0.1288,\n",
            "          0.0293,  1.0901,  1.5859,  0.0000,  1.9746,  0.3322]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.5951, 0.0000,  ..., 0.0000, 1.8296, 0.0000],\n",
            "        [0.0000, 1.8381, 0.0000,  ..., 0.0000, 1.8410, 0.0830],\n",
            "        [0.0000, 2.1003, 0.0000,  ..., 0.0000, 2.3854, 0.6203],\n",
            "        ...,\n",
            "        [0.0000, 1.3309, 0.9549,  ..., 0.0000, 2.5039, 0.1210],\n",
            "        [0.0000, 1.9882, 0.0000,  ..., 0.0000, 1.8330, 0.3620],\n",
            "        [0.0000, 1.4434, 0.9732,  ..., 0.0000, 2.5871, 0.1864]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5415],\n",
            "        [0.4902],\n",
            "        [0.4995],\n",
            "        ...,\n",
            "        [0.2812],\n",
            "        [0.5263],\n",
            "        [0.2828]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5736],\n",
            "        [0.5736],\n",
            "        [0.5736],\n",
            "        ...,\n",
            "        [0.5736],\n",
            "        [0.5736],\n",
            "        [0.5736]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4051, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4051, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4051], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 10\n",
            "node feats are tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[99, 95, 40,  ..., 10,  8, 84],\n",
            "        [87, 57, 58,  ...,  6, 68,  7],\n",
            "        [10, 40, 38,  ..., 55, 70, 24],\n",
            "        ...,\n",
            "        [55, 10, 52,  ..., 92, 52, 68],\n",
            "        [11, 63, 35,  ..., 55, 22,  2],\n",
            "        [45, 32, 86,  ..., 75, 33, 64]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-9.8815e-03,  3.3039e-02, -1.4396e-02,  ..., -2.4584e-01,\n",
            "         -1.9240e-01, -5.0579e-02],\n",
            "        [-7.3013e-03,  2.8815e-02,  5.1675e-03,  ..., -2.4697e-01,\n",
            "         -1.8811e-01, -5.2350e-02],\n",
            "        [ 9.3683e-04,  3.9771e-02, -1.1199e-02,  ..., -2.4728e-01,\n",
            "         -1.8636e-01, -5.8734e-02],\n",
            "        ...,\n",
            "        [-9.3918e-03,  3.7898e-02, -1.3051e-02,  ..., -2.5934e-01,\n",
            "         -1.8704e-01, -5.5233e-02],\n",
            "        [-1.6346e-03,  3.3215e-02, -4.2838e-03,  ..., -2.5820e-01,\n",
            "         -1.8774e-01, -5.5045e-02],\n",
            "        [-2.4624e-04,  3.6629e-02, -1.2746e-02,  ..., -2.6082e-01,\n",
            "         -1.9136e-01, -4.7527e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2425e+00,  8.5862e-03, -8.1997e-01,  ..., -1.0218e+00,\n",
            "          3.6856e-02, -7.1853e-01],\n",
            "        [-1.2397e+00, -3.1213e-03, -8.1182e-01,  ..., -1.0181e+00,\n",
            "          3.5175e-02, -7.1327e-01],\n",
            "        [-1.2385e+00,  5.3176e-03, -8.1309e-01,  ..., -1.0195e+00,\n",
            "          3.7302e-02, -7.1576e-01],\n",
            "        ...,\n",
            "        [-1.2375e+00,  1.0936e-02, -8.1970e-01,  ..., -1.0167e+00,\n",
            "          4.0497e-02, -7.1411e-01],\n",
            "        [-1.2467e+00,  2.2117e-04, -8.2000e-01,  ..., -1.0217e+00,\n",
            "          3.8443e-02, -7.1900e-01],\n",
            "        [-1.2513e+00,  3.0585e-04, -8.2132e-01,  ..., -1.0289e+00,\n",
            "          3.6626e-02, -7.1655e-01]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3687,  0.1140, -0.0203, -0.2542, -0.0555,  0.0730, -0.0617,  0.2055,\n",
            "          0.1482,  0.3005,  0.1110,  0.0044,  1.4937,  0.1718,  1.7926,  0.1041,\n",
            "          0.1630,  1.2492,  1.2123,  0.0000,  1.9955,  0.2153]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.3641, 0.0000,  ..., 0.0000, 1.7980, 0.2032],\n",
            "        [0.0000, 1.7414, 0.1192,  ..., 0.0000, 1.8532, 0.0000],\n",
            "        [0.0000, 1.5342, 0.0000,  ..., 0.0000, 1.8702, 0.1685],\n",
            "        ...,\n",
            "        [0.0000, 1.5622, 0.0000,  ..., 0.0000, 1.9796, 0.0952],\n",
            "        [0.0000, 1.8076, 0.0000,  ..., 0.0000, 1.9320, 1.2701],\n",
            "        [0.0000, 1.5584, 0.0000,  ..., 0.0000, 1.9691, 0.0895]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5073],\n",
            "        [0.4407],\n",
            "        [0.4627],\n",
            "        ...,\n",
            "        [0.4317],\n",
            "        [0.7971],\n",
            "        [0.4321]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5642],\n",
            "        [0.5642],\n",
            "        [0.5642],\n",
            "        ...,\n",
            "        [0.5642],\n",
            "        [0.5642],\n",
            "        [0.5642]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4044, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4044, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4044], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 11\n",
            "node feats are tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[49, 45, 47,  ..., 55, 12, 94],\n",
            "        [72, 30, 86,  ..., 55, 96, 33],\n",
            "        [59, 17, 93,  ..., 65, 58, 29],\n",
            "        ...,\n",
            "        [55, 29, 27,  ..., 29, 19, 29],\n",
            "        [14, 77, 70,  ..., 89, 10, 27],\n",
            "        [54,  4, 20,  ..., 30, 93, 88]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-3.3935e-03,  2.5090e-02, -4.5117e-04,  ..., -2.8810e-01,\n",
            "         -1.5706e-01, -6.5179e-02],\n",
            "        [-2.1893e-03,  1.8687e-02,  1.0505e-03,  ..., -2.8711e-01,\n",
            "         -1.7022e-01, -6.0682e-02],\n",
            "        [-7.1783e-03,  2.9874e-02, -7.4359e-03,  ..., -2.8535e-01,\n",
            "         -1.6961e-01, -5.6511e-02],\n",
            "        ...,\n",
            "        [ 4.5892e-03,  2.0043e-02, -6.6570e-03,  ..., -2.8627e-01,\n",
            "         -1.7359e-01, -6.6564e-02],\n",
            "        [ 6.2911e-03,  1.6562e-02,  8.6973e-05,  ..., -2.8935e-01,\n",
            "         -1.7181e-01, -6.4006e-02],\n",
            "        [-8.3408e-04,  2.5895e-02, -8.0816e-03,  ..., -2.8999e-01,\n",
            "         -1.6645e-01, -6.0376e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4351,  0.2604, -0.8288,  ..., -1.0227, -0.0075, -0.8235],\n",
            "        [-1.4299,  0.2681, -0.8250,  ..., -1.0179, -0.0140, -0.8212],\n",
            "        [-1.4282,  0.2630, -0.8220,  ..., -1.0155, -0.0104, -0.8231],\n",
            "        ...,\n",
            "        [-1.4272,  0.2622, -0.8264,  ..., -1.0187, -0.0070, -0.8315],\n",
            "        [-1.4354,  0.2616, -0.8294,  ..., -1.0178, -0.0110, -0.8249],\n",
            "        [-1.4358,  0.2587, -0.8254,  ..., -1.0190, -0.0106, -0.8310]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3882,  0.1214, -0.0161, -0.2607, -0.0537,  0.0769, -0.0602,  0.1619,\n",
            "          0.1442,  0.3773,  0.0353,  0.0060,  1.8676,  0.1628,  1.8835,  0.0799,\n",
            "          0.0587,  1.0589,  1.3502,  0.0000,  1.8874,  0.3264]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0393, 1.6048, 1.9853,  ..., 0.0000, 2.0700, 0.0000],\n",
            "        [0.0000, 1.9923, 0.0000,  ..., 0.0000, 1.7717, 0.3243],\n",
            "        [0.0000, 2.2027, 0.0000,  ..., 0.0000, 1.5188, 0.0936],\n",
            "        ...,\n",
            "        [0.0000, 2.0838, 0.1102,  ..., 0.0000, 1.7087, 0.0000],\n",
            "        [0.0000, 2.0701, 0.0000,  ..., 0.0000, 1.8379, 0.2637],\n",
            "        [0.2819, 0.6181, 1.3199,  ..., 0.0000, 2.0950, 0.2377]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.3612],\n",
            "        [0.5499],\n",
            "        [0.4877],\n",
            "        ...,\n",
            "        [0.5095],\n",
            "        [0.5184],\n",
            "        [0.3676]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5601],\n",
            "        [0.5601],\n",
            "        [0.5601],\n",
            "        ...,\n",
            "        [0.5601],\n",
            "        [0.5601],\n",
            "        [0.5601]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4068, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4068, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4068], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 12\n",
            "node feats are tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[56, 80, 66,  ..., 82, 60, 95],\n",
            "        [90, 34, 12,  ..., 61, 29, 65],\n",
            "        [80, 24, 79,  ..., 43, 86, 24],\n",
            "        ...,\n",
            "        [46, 60, 49,  ..., 40, 49, 69],\n",
            "        [39, 44, 87,  ..., 91, 16,  5],\n",
            "        [89, 30, 94,  ...,  5, 43, 34]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0140,  0.0229,  0.0157,  ..., -0.2756, -0.1662, -0.0406],\n",
            "        [-0.0081,  0.0294,  0.0141,  ..., -0.2701, -0.1759, -0.0454],\n",
            "        [-0.0034,  0.0208,  0.0102,  ..., -0.2732, -0.1704, -0.0458],\n",
            "        ...,\n",
            "        [-0.0117,  0.0217,  0.0158,  ..., -0.2770, -0.1738, -0.0462],\n",
            "        [-0.0146,  0.0207,  0.0082,  ..., -0.2752, -0.1753, -0.0407],\n",
            "        [-0.0102,  0.0267,  0.0133,  ..., -0.2701, -0.1762, -0.0421]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3165,  0.2344, -0.7533,  ..., -0.9609, -0.0321, -0.7453],\n",
            "        [-1.3279,  0.2429, -0.7621,  ..., -0.9635, -0.0337, -0.7481],\n",
            "        [-1.3229,  0.2359, -0.7556,  ..., -0.9612, -0.0334, -0.7482],\n",
            "        ...,\n",
            "        [-1.3370,  0.2466, -0.7665,  ..., -0.9735, -0.0376, -0.7467],\n",
            "        [-1.3274,  0.2312, -0.7557,  ..., -0.9654, -0.0339, -0.7404],\n",
            "        [-1.3222,  0.2260, -0.7546,  ..., -0.9599, -0.0326, -0.7482]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3887,  0.1055, -0.0183, -0.2588, -0.0546,  0.0762, -0.0445,  0.1693,\n",
            "          0.1455,  0.3702,  0.0799,  0.0066,  1.7924,  0.1670,  1.8969,  0.0822,\n",
            "          0.0795,  1.1382,  1.2340,  0.0000,  1.8046,  0.3581]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9161, 0.0000,  ..., 0.0000, 1.7519, 0.3160],\n",
            "        [0.0000, 1.8872, 0.0000,  ..., 0.0000, 1.6967, 0.5358],\n",
            "        [0.0000, 1.8140, 0.0000,  ..., 0.0000, 2.0288, 0.2004],\n",
            "        ...,\n",
            "        [0.0000, 2.8550, 0.0000,  ..., 0.0000, 1.9698, 1.2117],\n",
            "        [0.0000, 1.6124, 1.5956,  ..., 0.0000, 2.2952, 0.0000],\n",
            "        [0.0000, 1.8044, 0.0000,  ..., 0.0000, 2.0054, 0.0678]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5312],\n",
            "        [0.6152],\n",
            "        [0.5169],\n",
            "        ...,\n",
            "        [0.8411],\n",
            "        [0.2944],\n",
            "        [0.4545]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5429],\n",
            "        [0.5429],\n",
            "        [0.5429],\n",
            "        ...,\n",
            "        [0.5429],\n",
            "        [0.5429],\n",
            "        [0.5429]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4109, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4109, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4109], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 13\n",
            "node feats are tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[70,  1, 14,  ..., 84, 10, 79],\n",
            "        [34,  8, 67,  ..., 62, 93, 41],\n",
            "        [21, 54, 33,  ..., 10, 34, 24],\n",
            "        ...,\n",
            "        [40, 42, 74,  ..., 80, 21, 48],\n",
            "        [72, 40,  9,  ...,  0,  6,  1],\n",
            "        [65, 96, 26,  ..., 59, 13, 79]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0210, -0.0013,  0.0130,  ..., -0.2544, -0.1580, -0.0400],\n",
            "        [ 0.0208,  0.0026,  0.0126,  ..., -0.2717, -0.1645, -0.0477],\n",
            "        [ 0.0278, -0.0034,  0.0153,  ..., -0.2689, -0.1616, -0.0466],\n",
            "        ...,\n",
            "        [ 0.0250,  0.0018,  0.0124,  ..., -0.2609, -0.1552, -0.0529],\n",
            "        [ 0.0249, -0.0009,  0.0112,  ..., -0.2722, -0.1583, -0.0420],\n",
            "        [ 0.0107,  0.0060,  0.0163,  ..., -0.2568, -0.1611, -0.0411]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3503,  0.2710, -0.7535,  ..., -0.9278,  0.1069, -0.7248],\n",
            "        [-1.3534,  0.2693, -0.7550,  ..., -0.9262,  0.1052, -0.7179],\n",
            "        [-1.3645,  0.2778, -0.7620,  ..., -0.9327,  0.1036, -0.7251],\n",
            "        ...,\n",
            "        [-1.3624,  0.2799, -0.7595,  ..., -0.9311,  0.1030, -0.7188],\n",
            "        [-1.3645,  0.2783, -0.7660,  ..., -0.9330,  0.1106, -0.7271],\n",
            "        [-1.3559,  0.2682, -0.7596,  ..., -0.9274,  0.1121, -0.7355]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 4.1152e-01,  1.4818e-01, -1.4965e-02, -2.7457e-01, -5.7571e-02,\n",
            "          9.2675e-02, -8.7091e-02,  1.4977e-01,  1.6753e-01,  3.6694e-01,\n",
            "          3.5789e-02,  4.0718e-04,  1.9474e+00,  1.4785e-01,  2.0806e+00,\n",
            "          1.1068e-01,  9.7104e-02,  9.4508e-01,  1.2895e+00,  0.0000e+00,\n",
            "          2.2505e+00,  3.1109e-01]], device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7126, 0.0000,  ..., 0.0000, 2.8877, 0.8566],\n",
            "        [0.0000, 1.6985, 0.0000,  ..., 0.0000, 1.8718, 0.0454],\n",
            "        [0.0000, 1.2938, 0.5583,  ..., 0.0000, 2.7287, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.0650, 0.0000,  ..., 0.0000, 2.2081, 0.4092],\n",
            "        [0.0000, 2.0455, 0.0000,  ..., 0.0000, 2.2222, 0.1849],\n",
            "        [0.0000, 1.7262, 0.2395,  ..., 0.0000, 1.7070, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5558],\n",
            "        [0.5232],\n",
            "        [0.3227],\n",
            "        ...,\n",
            "        [0.5915],\n",
            "        [0.5041],\n",
            "        [0.4756]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5831],\n",
            "        [0.5831],\n",
            "        [0.5831],\n",
            "        ...,\n",
            "        [0.5831],\n",
            "        [0.5831],\n",
            "        [0.5831]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4072, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4072, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4072], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 14\n",
            "node feats are tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 51, 16,  ..., 92, 75, 94],\n",
            "        [33, 94, 73,  ..., 65, 32, 53],\n",
            "        [81, 36, 36,  ..., 46, 86, 45],\n",
            "        ...,\n",
            "        [54, 48, 37,  ..., 99, 32, 77],\n",
            "        [91, 91, 45,  ..., 39, 69, 84],\n",
            "        [ 9, 39, 24,  ..., 25, 73, 57]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0023, -0.0040,  0.0098,  ..., -0.2605, -0.1867, -0.0652],\n",
            "        [ 0.0024, -0.0030,  0.0061,  ..., -0.2734, -0.1827, -0.0752],\n",
            "        [ 0.0059, -0.0104,  0.0045,  ..., -0.2712, -0.1833, -0.0646],\n",
            "        ...,\n",
            "        [ 0.0008, -0.0032,  0.0054,  ..., -0.2682, -0.1822, -0.0684],\n",
            "        [-0.0071, -0.0059, -0.0010,  ..., -0.2691, -0.1834, -0.0662],\n",
            "        [ 0.0089, -0.0031,  0.0118,  ..., -0.2567, -0.1881, -0.0609]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2458,  0.0914, -0.6628,  ..., -0.8577,  0.0235, -0.8149],\n",
            "        [-1.2451,  0.0951, -0.6645,  ..., -0.8539,  0.0264, -0.8169],\n",
            "        [-1.2447,  0.1061, -0.6587,  ..., -0.8499,  0.0168, -0.7993],\n",
            "        ...,\n",
            "        [-1.2484,  0.0970, -0.6599,  ..., -0.8496,  0.0214, -0.8099],\n",
            "        [-1.2418,  0.0943, -0.6605,  ..., -0.8481,  0.0191, -0.8070],\n",
            "        [-1.2433,  0.0934, -0.6598,  ..., -0.8512,  0.0212, -0.8038]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3896,  0.1156, -0.0154, -0.2598, -0.0480,  0.0770, -0.0724,  0.1873,\n",
            "          0.1439,  0.3984,  0.0102,  0.0048,  1.9519,  0.1832,  2.0987,  0.0753,\n",
            "          0.0879,  1.1989,  1.4878,  0.0000,  1.9381,  0.2261]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.2459, 0.0000,  ..., 0.0000, 1.5210, 0.2694],\n",
            "        [0.0000, 1.8341, 0.0000,  ..., 0.0000, 1.5757, 0.2071],\n",
            "        [0.0000, 2.0982, 0.1088,  ..., 0.0000, 1.8355, 0.2453],\n",
            "        ...,\n",
            "        [0.0000, 1.7610, 0.0000,  ..., 0.0000, 1.3431, 0.0000],\n",
            "        [0.0000, 1.4687, 0.0000,  ..., 0.0000, 1.4244, 0.0000],\n",
            "        [0.0000, 1.9065, 0.0000,  ..., 0.0000, 1.7338, 0.2307]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5294],\n",
            "        [0.5292],\n",
            "        [0.5274],\n",
            "        ...,\n",
            "        [0.5054],\n",
            "        [0.4734],\n",
            "        [0.5242]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5781],\n",
            "        [0.5781],\n",
            "        [0.5781],\n",
            "        ...,\n",
            "        [0.5781],\n",
            "        [0.5781],\n",
            "        [0.5781]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4058, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4058, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4058], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 15\n",
            "node feats are tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 9, 26, 36,  ..., 82, 76, 80],\n",
            "        [20, 94, 40,  ..., 66, 67, 60],\n",
            "        [85, 46, 58,  ..., 98, 75, 58],\n",
            "        ...,\n",
            "        [73, 46, 37,  ..., 63, 11, 11],\n",
            "        [92, 76,  6,  ..., 46, 68, 23],\n",
            "        [53, 82, 25,  ..., 16, 29, 32]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.0345e-02,  1.7116e-02,  1.4892e-02,  ..., -2.7669e-01,\n",
            "         -1.7663e-01, -5.3528e-02],\n",
            "        [-1.1132e-02,  2.6139e-02,  2.1191e-02,  ..., -2.8098e-01,\n",
            "         -1.8646e-01, -5.1669e-02],\n",
            "        [-8.5590e-03,  1.7850e-02,  1.2117e-02,  ..., -2.6957e-01,\n",
            "         -1.8372e-01, -5.7893e-02],\n",
            "        ...,\n",
            "        [-1.2468e-02,  1.4315e-02,  2.5929e-02,  ..., -2.7241e-01,\n",
            "         -1.7743e-01, -4.7237e-02],\n",
            "        [-9.3355e-03,  1.5877e-02,  1.9265e-02,  ..., -2.6792e-01,\n",
            "         -1.8129e-01, -5.1510e-02],\n",
            "        [-4.9720e-06,  1.0993e-02,  1.3731e-02,  ..., -2.8490e-01,\n",
            "         -1.8148e-01, -5.1090e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2827,  0.0868, -0.7216,  ..., -0.9901,  0.0082, -0.8369],\n",
            "        [-1.2762,  0.0970, -0.7215,  ..., -0.9896,  0.0064, -0.8299],\n",
            "        [-1.2941,  0.1004, -0.7307,  ..., -1.0041,  0.0056, -0.8378],\n",
            "        ...,\n",
            "        [-1.2770,  0.0862, -0.7217,  ..., -0.9916,  0.0118, -0.8422],\n",
            "        [-1.2836,  0.0849, -0.7215,  ..., -0.9948,  0.0097, -0.8339],\n",
            "        [-1.2852,  0.0900, -0.7247,  ..., -0.9943,  0.0048, -0.8296]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3557,  0.1303, -0.0138, -0.2493, -0.0469,  0.0633, -0.0638,  0.1880,\n",
            "          0.1221,  0.3729, -0.0080,  0.0106,  1.7811,  0.2383,  1.7240,  0.0491,\n",
            "          0.0502,  1.0680,  1.5703,  0.0000,  1.7456,  0.2063]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9424, 0.4746,  ..., 0.0000, 2.2065, 0.1316],\n",
            "        [0.0000, 2.9027, 0.0000,  ..., 0.0000, 1.7436, 0.7233],\n",
            "        [0.0000, 1.0449, 0.2933,  ..., 0.0000, 1.2587, 0.3669],\n",
            "        ...,\n",
            "        [0.0000, 1.4572, 0.0000,  ..., 0.0000, 2.0747, 0.3579],\n",
            "        [0.0000, 1.9946, 0.0330,  ..., 0.0000, 1.7113, 0.1213],\n",
            "        [0.0000, 1.6784, 0.0000,  ..., 0.0000, 1.5069, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5248],\n",
            "        [0.6718],\n",
            "        [0.5156],\n",
            "        ...,\n",
            "        [0.4505],\n",
            "        [0.4503],\n",
            "        [0.4547]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5857],\n",
            "        [0.5857],\n",
            "        [0.5857],\n",
            "        ...,\n",
            "        [0.5857],\n",
            "        [0.5857],\n",
            "        [0.5857]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4018, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4018, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4018], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 16\n",
            "node feats are tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[69, 65, 15,  ..., 17, 69, 10],\n",
            "        [99, 38, 25,  ..., 64, 27, 96],\n",
            "        [64, 79, 86,  ..., 35, 49, 69],\n",
            "        ...,\n",
            "        [62, 28, 96,  ..., 32, 95, 16],\n",
            "        [42, 80, 40,  ..., 11, 77, 54],\n",
            "        [23,  9, 55,  ..., 71, 86, 55]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-0.0343, -0.0017, -0.0034,  ..., -0.2641, -0.1746, -0.0809],\n",
            "        [-0.0314, -0.0017,  0.0098,  ..., -0.2721, -0.1769, -0.0717],\n",
            "        [-0.0361, -0.0061,  0.0198,  ..., -0.2719, -0.1770, -0.0755],\n",
            "        ...,\n",
            "        [-0.0295, -0.0057,  0.0061,  ..., -0.2645, -0.1696, -0.0602],\n",
            "        [-0.0303, -0.0037,  0.0083,  ..., -0.2785, -0.1752, -0.0723],\n",
            "        [-0.0408, -0.0042,  0.0093,  ..., -0.2642, -0.1743, -0.0605]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2151,  0.1192, -0.5430,  ..., -0.9518,  0.0183, -0.7578],\n",
            "        [-1.2168,  0.1183, -0.5432,  ..., -0.9548,  0.0195, -0.7566],\n",
            "        [-1.2094,  0.1140, -0.5396,  ..., -0.9507,  0.0177, -0.7514],\n",
            "        ...,\n",
            "        [-1.2107,  0.1097, -0.5412,  ..., -0.9497,  0.0197, -0.7577],\n",
            "        [-1.2060,  0.1152, -0.5412,  ..., -0.9504,  0.0171, -0.7513],\n",
            "        [-1.2199,  0.1230, -0.5449,  ..., -0.9522,  0.0152, -0.7538]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3294,  0.1470, -0.0160, -0.2469, -0.0476,  0.0619, -0.0622,  0.2473,\n",
            "          0.1164,  0.3303,  0.0897,  0.0115,  1.4217,  0.4056,  1.8030,  0.1357,\n",
            "          0.0754,  1.2082,  1.5469,  0.0000,  1.8126,  0.1999]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.9767, 0.0000,  ..., 0.0000, 1.5548, 0.9162],\n",
            "        [0.0488, 0.6963, 0.0000,  ..., 0.0000, 0.9172, 0.0000],\n",
            "        [0.0000, 0.8478, 0.2755,  ..., 0.0000, 1.8024, 0.4171],\n",
            "        ...,\n",
            "        [0.0000, 1.4542, 0.3925,  ..., 0.0000, 1.7486, 0.0946],\n",
            "        [0.0000, 2.4559, 0.4049,  ..., 0.0000, 2.8102, 0.8744],\n",
            "        [0.0000, 2.1662, 0.4785,  ..., 0.0000, 1.8876, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.7767],\n",
            "        [0.5140],\n",
            "        [0.4754],\n",
            "        ...,\n",
            "        [0.4191],\n",
            "        [0.4027],\n",
            "        [0.4950]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5955],\n",
            "        [0.5955],\n",
            "        [0.5955],\n",
            "        ...,\n",
            "        [0.5955],\n",
            "        [0.5955],\n",
            "        [0.5955]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4005, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4005, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4005], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 17\n",
            "node feats are tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[ 8, 36, 88,  ..., 58, 32, 66],\n",
            "        [10, 54,  0,  ..., 52, 44, 89],\n",
            "        [76, 39, 58,  ..., 42, 37, 60],\n",
            "        ...,\n",
            "        [ 6,  8, 80,  ..., 33, 20, 73],\n",
            "        [ 7, 95, 78,  ...,  9, 49, 68],\n",
            "        [26, 92, 66,  ..., 44, 78, 62]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 0.0079, -0.0212, -0.0004,  ..., -0.2792, -0.2098, -0.0352],\n",
            "        [ 0.0211, -0.0158,  0.0010,  ..., -0.2837, -0.1940, -0.0393],\n",
            "        [ 0.0083, -0.0143,  0.0075,  ..., -0.2848, -0.1954, -0.0441],\n",
            "        ...,\n",
            "        [ 0.0068, -0.0136,  0.0021,  ..., -0.2844, -0.2009, -0.0408],\n",
            "        [ 0.0051, -0.0236,  0.0120,  ..., -0.2777, -0.1934, -0.0256],\n",
            "        [ 0.0149, -0.0121,  0.0044,  ..., -0.2873, -0.1965, -0.0454]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.4603,  0.1159, -0.7609,  ..., -1.1019,  0.0578, -0.7507],\n",
            "        [-1.4686,  0.1135, -0.7589,  ..., -1.1076,  0.0559, -0.7580],\n",
            "        [-1.4622,  0.0997, -0.7536,  ..., -1.0974,  0.0643, -0.7618],\n",
            "        ...,\n",
            "        [-1.4597,  0.1038, -0.7524,  ..., -1.0992,  0.0605, -0.7574],\n",
            "        [-1.4718,  0.1134, -0.7613,  ..., -1.1081,  0.0638, -0.7674],\n",
            "        [-1.4576,  0.1195, -0.7552,  ..., -1.1016,  0.0564, -0.7527]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3775,  0.1007, -0.0176, -0.2518, -0.0519,  0.0699, -0.0575,  0.1833,\n",
            "          0.1421,  0.3726,  0.0509,  0.0085,  1.7589,  0.1290,  1.8990,  0.0649,\n",
            "          0.0908,  1.2468,  1.3855,  0.0000,  1.8387,  0.3033]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 1.7506, 0.0000,  ..., 0.0000, 1.6709, 0.2577],\n",
            "        [0.0000, 1.8387, 0.0000,  ..., 0.0000, 1.4770, 0.0000],\n",
            "        [0.0000, 2.6835, 0.0000,  ..., 0.0000, 2.3186, 0.7591],\n",
            "        ...,\n",
            "        [0.0000, 1.8496, 0.0000,  ..., 0.0000, 1.8310, 0.1988],\n",
            "        [0.0000, 1.8646, 0.0000,  ..., 0.0000, 1.7731, 0.2397],\n",
            "        [0.0000, 0.8755, 0.0000,  ..., 0.0000, 1.6102, 0.1868]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5325],\n",
            "        [0.4642],\n",
            "        [0.7348],\n",
            "        ...,\n",
            "        [0.4699],\n",
            "        [0.4809],\n",
            "        [0.4462]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5471],\n",
            "        [0.5471],\n",
            "        [0.5471],\n",
            "        ...,\n",
            "        [0.5471],\n",
            "        [0.5471],\n",
            "        [0.5471]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4086, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4086, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4086], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 18\n",
            "node feats are tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[39, 61, 65,  ..., 72, 63, 87],\n",
            "        [18, 79, 54,  ..., 16, 55,  0],\n",
            "        [60, 85, 42,  ..., 80, 55, 27],\n",
            "        ...,\n",
            "        [40, 81, 72,  ..., 49, 72,  9],\n",
            "        [38,  8, 78,  ...,  8, 31, 70],\n",
            "        [43, 77, 22,  ..., 76, 10,  1]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[ 1.0053e-02, -4.4840e-03,  2.6774e-02,  ..., -2.6715e-01,\n",
            "         -1.6322e-01, -4.8297e-02],\n",
            "        [ 4.5920e-03, -5.4925e-03,  2.0619e-02,  ..., -2.7071e-01,\n",
            "         -1.6295e-01, -4.0090e-02],\n",
            "        [-1.5486e-04,  1.0930e-03,  1.2937e-02,  ..., -2.6371e-01,\n",
            "         -1.6834e-01, -4.9408e-02],\n",
            "        ...,\n",
            "        [-7.3286e-04, -7.4764e-04,  1.5761e-02,  ..., -2.7402e-01,\n",
            "         -1.7375e-01, -4.6776e-02],\n",
            "        [ 9.2543e-03, -1.1744e-02,  1.3695e-02,  ..., -2.6015e-01,\n",
            "         -1.6775e-01, -4.0177e-02],\n",
            "        [ 8.1152e-03, -8.4247e-03,  1.6970e-02,  ..., -2.6800e-01,\n",
            "         -1.7722e-01, -4.2522e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.3009,  0.2618, -0.7968,  ..., -0.9924,  0.0043, -0.8756],\n",
            "        [-1.3075,  0.2623, -0.8068,  ..., -0.9988,  0.0072, -0.8912],\n",
            "        [-1.3043,  0.2571, -0.7971,  ..., -0.9918,  0.0092, -0.8848],\n",
            "        ...,\n",
            "        [-1.3090,  0.2548, -0.8036,  ..., -1.0026,  0.0142, -0.8978],\n",
            "        [-1.2945,  0.2573, -0.7928,  ..., -0.9868,  0.0178, -0.8886],\n",
            "        [-1.2990,  0.2572, -0.7941,  ..., -0.9859,  0.0107, -0.8842]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3622,  0.1333, -0.0122, -0.2533, -0.0456,  0.0657, -0.0918,  0.1777,\n",
            "          0.1288,  0.3816, -0.0892,  0.0064,  1.9220,  0.1384,  1.7866,  0.0530,\n",
            "          0.0198,  1.0832,  1.6901,  0.0000,  1.9265,  0.0947]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.4116, 0.0000,  ..., 0.0000, 1.8564, 0.0000],\n",
            "        [0.0000, 2.0797, 0.0000,  ..., 0.0000, 1.8811, 0.0000],\n",
            "        [0.0000, 1.6581, 0.0000,  ..., 0.0000, 2.1583, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 1.4116, 0.0000,  ..., 0.0000, 1.2814, 0.0000],\n",
            "        [0.0000, 2.0185, 0.0000,  ..., 0.0000, 1.8049, 0.0715],\n",
            "        [0.0000, 1.9210, 0.0000,  ..., 0.0000, 1.7998, 0.0000]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.5397],\n",
            "        [0.4248],\n",
            "        [0.3634],\n",
            "        ...,\n",
            "        [0.4779],\n",
            "        [0.4636],\n",
            "        [0.4967]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5993],\n",
            "        [0.5993],\n",
            "        [0.5993],\n",
            "        ...,\n",
            "        [0.5993],\n",
            "        [0.5993],\n",
            "        [0.5993]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.3971, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.3971, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.3971], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "index of dataloader is 19\n",
            "node feats are tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11]) <class 'torch.Tensor'>\n",
            "x before emb is ->  tensor([[63, 47,  4,  ...,  3, 69, 15],\n",
            "        [97, 86, 74,  ..., 63, 50, 41],\n",
            "        [73, 59,  4,  ..., 56, 79, 73],\n",
            "        ...,\n",
            "        [81, 41, 35,  ..., 72, 40, 10],\n",
            "        [12, 52,  8,  ..., 65, 45, 32],\n",
            "        [78, 63, 55,  ..., 36, 80, 50]], device='cuda:0') torch.Size([3000, 11])\n",
            "x after emb. is ->  torch.Size([3000, 1100])\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-7.2537e-04,  3.7635e-03,  3.6579e-02,  ..., -2.8281e-01,\n",
            "         -1.8498e-01, -3.9943e-02],\n",
            "        [ 4.0383e-03,  2.3655e-03,  2.5419e-02,  ..., -2.7559e-01,\n",
            "         -1.8672e-01, -4.1113e-02],\n",
            "        [ 3.3130e-04, -3.3596e-04,  3.9271e-02,  ..., -2.7784e-01,\n",
            "         -1.8972e-01, -4.9652e-02],\n",
            "        ...,\n",
            "        [-7.5502e-05, -9.7309e-04,  3.3610e-02,  ..., -2.7679e-01,\n",
            "         -1.8110e-01, -2.5214e-02],\n",
            "        [-4.2507e-03,  7.0511e-03,  3.8018e-02,  ..., -2.8910e-01,\n",
            "         -1.8359e-01, -4.0872e-02],\n",
            "        [ 1.0141e-02,  3.8791e-03,  3.6868e-02,  ..., -2.8111e-01,\n",
            "         -1.8262e-01, -5.4999e-02]], device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "adj_matrix shape is ->  torch.Size([3000, 3000])\n",
            "head_outputs is  tensor([[-1.2425,  0.1890, -0.7911,  ..., -0.9707, -0.1989, -0.8181],\n",
            "        [-1.2424,  0.2012, -0.7955,  ..., -0.9734, -0.2058, -0.8132],\n",
            "        [-1.2499,  0.2024, -0.7956,  ..., -0.9724, -0.2094, -0.8062],\n",
            "        ...,\n",
            "        [-1.2527,  0.1903, -0.7937,  ..., -0.9758, -0.2053, -0.8124],\n",
            "        [-1.2346,  0.1911, -0.7877,  ..., -0.9675, -0.2037, -0.8077],\n",
            "        [-1.2557,  0.1957, -0.7981,  ..., -0.9728, -0.2059, -0.8109]],\n",
            "       device='cuda:0', grad_fn=<MeanBackward1>) torch.Size([3000, 24]) <class 'torch.Tensor'>\n",
            "shapes are ->  3000   11\n",
            "aggregator gives ->  tensor([[ 0.3673,  0.1115, -0.0145, -0.2509, -0.0457,  0.0707, -0.0661,  0.1811,\n",
            "          0.1297,  0.3950, -0.0156,  0.0051,  1.8991,  0.1548,  1.8926,  0.0935,\n",
            "          0.0298,  1.1345,  1.5472,  0.0000,  1.7350,  0.2025]],\n",
            "       device='cuda:0', grad_fn=<CatBackward0>) torch.Size([1, 22])\n",
            "encoder gives ->  tensor([[0.0000, 2.9443, 0.0000,  ..., 0.0000, 1.7774, 1.0754],\n",
            "        [0.0000, 2.0318, 0.0000,  ..., 0.0000, 1.7094, 0.1054],\n",
            "        [0.1966, 1.5131, 0.6895,  ..., 0.0000, 1.9435, 0.0000],\n",
            "        ...,\n",
            "        [0.0000, 2.9462, 0.0000,  ..., 0.0000, 1.8436, 0.9483],\n",
            "        [0.0000, 1.2840, 0.8481,  ..., 0.0000, 2.2643, 0.0227],\n",
            "        [0.0000, 2.1625, 0.0000,  ..., 0.0000, 1.4195, 0.0229]],\n",
            "       device='cuda:0', grad_fn=<ReluBackward0>) torch.Size([3000, 11])\n",
            "local_output tensor([[0.7897],\n",
            "        [0.4668],\n",
            "        [0.3310],\n",
            "        ...,\n",
            "        [0.7662],\n",
            "        [0.3061],\n",
            "        [0.4584]], device='cuda:0', grad_fn=<SigmoidBackward0>) torch.Size([3000, 1]) <class 'torch.Tensor'>\n",
            "global_output tensor([[0.5723],\n",
            "        [0.5723],\n",
            "        [0.5723],\n",
            "        ...,\n",
            "        [0.5723],\n",
            "        [0.5723],\n",
            "        [0.5723]], device='cuda:0') torch.Size([3000, 1]) <class 'torch.Tensor'> 1\n",
            "num of nodes and num of graphs are ->  3000   3000\n",
            "loss_init is:  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>) torch.Size([]) <class 'torch.Tensor'>\n",
            "loss is  tensor(1.4041, device='cuda:0', grad_fn=<AddBackward0>)   torch.Size([])   <class 'torch.Tensor'>\n",
            "loss final  tensor([1.4041], requires_grad=True) torch.Size([1]) <class 'torch.Tensor'>\n",
            "===== Epoch 6, Loss tensor([28.1186], grad_fn=<DivBackward0>) =====\n"
          ]
        }
      ],
      "source": [
        "#main training loop\n",
        "log_interval = 5\n",
        "for epoch in range(1, epochs+1):\n",
        "    loss_all = 0\n",
        "    device = next(model.parameters()).device\n",
        "    model.train()\n",
        "    for index,data in enumerate(dataloader):\n",
        "      data = data.to(device)\n",
        "      optimizer.zero_grad()\n",
        "      print(\"index of dataloader is\" , index)\n",
        "      print(\"node feats are\",data.x,np.shape(data.x),type(data.x))\n",
        "      local_d, global_d = model(data.x, data.edge_index, data.edge_attr)\n",
        "      global_d = global_d.item()*torch.ones(num_nodes,1).to(device)\n",
        "      discr_global = torch.unique(global_d).numel()\n",
        "      print(\"local_output\",local_d,np.shape(local_d),type(local_d))\n",
        "      print(\"global_output\",global_d,np.shape(global_d),type(global_d),discr_global)\n",
        "      loss = model.local_global_loss_(local_d,global_d,data.edge_index,data.batch)\n",
        "      # loss = model.probability_loss(local_d,global_d)\n",
        "      loss_batch = loss\n",
        "      print(\"loss_init is: \",loss,np.shape(loss),type(loss))\n",
        "      print(\"loss is \", loss,\" \",np.shape(loss),\" \",type(loss))\n",
        "      loss_final = torch.tensor([loss_batch.item()],requires_grad=True)\n",
        "      loss_all += loss_final*num_graphs\n",
        "      print(\"loss final \", loss_final,np.shape(loss_final),type(loss_final))\n",
        "      loss_final.backward()\n",
        "      optimizer.step()\n",
        "    print('===== Epoch {}, Loss {} ====='.format(epoch, loss_all / len(dataloader)))\n",
        "    if epoch % log_interval == 0:\n",
        "      print(\"mpika evaluate mode!!\")\n",
        "      model.eval()\n",
        "      data = data.to(device)\n",
        "      local_d, global_d = model(data.x,data.edge_index,data.edge_attr)\n",
        "      print(\"local_scores are -> \",local_d,\" \",np.shape(local_d),\" at epoch: \",epoch)\n",
        "      print(\"global_scores are -> \",global_d,\" \",np.shape(global_d),\" at epoch: \",epoch)\n",
        "torch.save(model.state_dict(), '/content/drive/My Drive/model.pth')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L1RMj8hTgPOn"
      },
      "outputs": [],
      "source": [
        "with torch.no_grad():\n",
        "    torch.cuda.empty_cache()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}