corpus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1213 1064\n"
     ]
    }
   ],
   "source": [
    "# merge duplicated sentences in templates.txt\n",
    "file = open('./corpus/templates.txt', 'r', encoding='utf-8')\n",
    "templates_txt = file.read()\n",
    "file.close()\n",
    "sentences = templates_txt.split('\\n')\n",
    "sentences = [s for s in sentences if s]\n",
    "\n",
    "unique_sentences = sorted(set(sentences))\n",
    "\n",
    "print(len(sentences), len(unique_sentences))\n",
    "\n",
    "new_txt = '\\n'.join(unique_sentences)\n",
    "\n",
    "ofile = open('./corpus/templates.txt', 'w', encoding='utf-8')\n",
    "ofile.write(new_txt)\n",
    "ofile.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('of', 85),\n",
       " ('detailed', 70),\n",
       " ('a', 61),\n",
       " ('and', 58),\n",
       " ('the', 52),\n",
       " ('lighting', 51),\n",
       " ('intricate', 43),\n",
       " ('by', 43),\n",
       " ('portrait', 40),\n",
       " ('in', 38),\n",
       " ('realistic', 37),\n",
       " ('art', 35),\n",
       " ('8k', 35),\n",
       " ('cinematic', 34),\n",
       " ('beautiful', 32),\n",
       " ('render', 30),\n",
       " ('octane', 29),\n",
       " ('with', 25),\n",
       " ('engine', 25),\n",
       " ('unreal', 25),\n",
       " ('highly', 22),\n",
       " ('light', 22),\n",
       " ('fantasy', 21),\n",
       " ('style', 20),\n",
       " ('detail', 19),\n",
       " ('design', 19),\n",
       " ('on', 19),\n",
       " ('concept', 17),\n",
       " ('volumetric', 17),\n",
       " ('painting', 17),\n",
       " ('epic', 16),\n",
       " ('4k', 16),\n",
       " ('face', 16),\n",
       " ('dark', 15),\n",
       " ('photo', 15),\n",
       " ('hyper', 15),\n",
       " ('ultra', 14),\n",
       " ('photorealistic', 14),\n",
       " ('armor', 13),\n",
       " ('high', 13),\n",
       " ('details', 13),\n",
       " ('full', 13),\n",
       " ('white', 13),\n",
       " ('blue', 13),\n",
       " ('up', 12),\n",
       " ('eyes', 12),\n",
       " ('digital', 12),\n",
       " ('body', 11),\n",
       " ('an', 11),\n",
       " ('illustration', 11),\n",
       " ('woman', 10),\n",
       " ('black', 10),\n",
       " ('artgerm', 10),\n",
       " ('red', 10),\n",
       " ('soft', 9),\n",
       " ('skin', 9),\n",
       " ('sharp', 9),\n",
       " ('artstation', 9),\n",
       " ('5', 9),\n",
       " ('ornate', 9),\n",
       " ('finely', 9),\n",
       " ('book', 9),\n",
       " ('hair', 9),\n",
       " ('angle', 9),\n",
       " ('focus', 8),\n",
       " ('dramatic', 8),\n",
       " ('from', 8),\n",
       " ('ghibli', 8),\n",
       " ('extremely', 8),\n",
       " ('filigree', 8),\n",
       " ('photography', 8),\n",
       " ('greg', 8),\n",
       " ('rutkowski', 8),\n",
       " ('close', 8),\n",
       " ('girl', 8),\n",
       " ('paint', 7),\n",
       " ('background', 7),\n",
       " ('post', 7),\n",
       " ('water', 7),\n",
       " ('ethereal', 7),\n",
       " ('very', 7),\n",
       " ('green', 7),\n",
       " ('flowers', 7),\n",
       " ('ink', 7),\n",
       " ('female', 7),\n",
       " ('texture', 7),\n",
       " ('smooth', 7),\n",
       " ('shot', 7),\n",
       " ('to', 6),\n",
       " ('covered', 6),\n",
       " ('processing', 6),\n",
       " ('mucha', 6),\n",
       " ('anime', 6),\n",
       " ('animated', 6),\n",
       " ('as', 6),\n",
       " ('makoto', 6),\n",
       " ('anthropomorphic', 6),\n",
       " ('shinkai', 6),\n",
       " ('sky', 6),\n",
       " ('alphonse', 6),\n",
       " ('brown', 6),\n",
       " ('hyperrealistic', 6),\n",
       " ('painterly', 6),\n",
       " ('helmet', 6),\n",
       " ('burmese', 6),\n",
       " ('pixar', 6),\n",
       " ('dripping', 6),\n",
       " ('night', 5),\n",
       " ('old', 5),\n",
       " ('super', 5),\n",
       " ('paper', 5),\n",
       " ('stunning', 5),\n",
       " ('character', 5),\n",
       " ('silver', 5),\n",
       " ('warrior', 5),\n",
       " ('big', 5),\n",
       " ('nausicaa', 5),\n",
       " ('scene', 5),\n",
       " ('cute', 5),\n",
       " ('jean', 5),\n",
       " ('forest', 5),\n",
       " ('oil', 5),\n",
       " ('3d', 5),\n",
       " ('andreas', 5),\n",
       " ('studio', 5),\n",
       " ('knight', 5),\n",
       " ('miyazaki', 5),\n",
       " ('elegant', 5),\n",
       " ('rocha', 5),\n",
       " ('low', 5),\n",
       " ('pretty', 5),\n",
       " ('watercolor', 5),\n",
       " ('smoke', 5),\n",
       " ('hdr', 5),\n",
       " ('cover', 5),\n",
       " ('baptiste', 4),\n",
       " ('composition', 4),\n",
       " ('granular', 4),\n",
       " ('house', 4),\n",
       " ('metal', 4),\n",
       " ('asher', 4),\n",
       " ('photograph', 4),\n",
       " ('gasmask', 4),\n",
       " ('half', 4),\n",
       " ('realm', 4),\n",
       " ('young', 4),\n",
       " ('symmetrical', 4),\n",
       " ('monge', 4),\n",
       " ('trending', 4),\n",
       " ('japanese', 4),\n",
       " ('giger', 4),\n",
       " ('leaves', 4),\n",
       " ('renato', 4),\n",
       " ('god', 4),\n",
       " ('splatter', 4),\n",
       " ('painted', 4),\n",
       " ('comic', 4),\n",
       " ('ross', 4),\n",
       " ('profile', 4),\n",
       " ('wide', 4),\n",
       " ('view', 4),\n",
       " ('lush', 4),\n",
       " ('durand', 4),\n",
       " ('city', 4),\n",
       " ('immersive', 4),\n",
       " ('stone', 4),\n",
       " ('colors', 4),\n",
       " ('contrast', 4),\n",
       " ('architecture', 4),\n",
       " ('perfect', 4),\n",
       " ('is', 4),\n",
       " ('mucillo', 4),\n",
       " ('crystal', 4),\n",
       " ('golden', 4),\n",
       " ('pirate', 4),\n",
       " ('magic', 3),\n",
       " ('trees', 3),\n",
       " ('angry', 3),\n",
       " ('liquid', 3),\n",
       " ('wild', 3),\n",
       " ('cat', 3),\n",
       " ('equal', 3),\n",
       " ('s', 3),\n",
       " ('resolution', 3),\n",
       " ('lord', 3),\n",
       " ('bokeh', 3),\n",
       " ('glass', 3),\n",
       " ('rays', 3),\n",
       " ('princess', 3),\n",
       " ('bernie', 3),\n",
       " ('castle', 3),\n",
       " ('shadows', 3),\n",
       " ('tree', 3),\n",
       " ('apartment', 3),\n",
       " ('rococo', 3),\n",
       " ('legends', 3),\n",
       " ('snow', 3),\n",
       " ('fine', 3),\n",
       " ('john', 3),\n",
       " ('rendering', 3),\n",
       " ('fish', 3),\n",
       " ('r', 3),\n",
       " ('gate', 3),\n",
       " ('romance', 3),\n",
       " ('majestic', 3),\n",
       " ('rim', 3),\n",
       " ('village', 3),\n",
       " ('wrightson', 3),\n",
       " ('starry', 3),\n",
       " ('elf', 3),\n",
       " ('mystical', 3),\n",
       " ('small', 3),\n",
       " ('justice', 3),\n",
       " ('pink', 3),\n",
       " ('elve', 3),\n",
       " ('magical', 3),\n",
       " ('punk', 3),\n",
       " ('clothing', 3),\n",
       " ('hyperreal', 3),\n",
       " ('attractive', 3),\n",
       " ('plants', 3),\n",
       " ('adorable', 3),\n",
       " ('drawn', 3),\n",
       " ('ancient', 3),\n",
       " ('library', 3),\n",
       " ('breath', 3),\n",
       " ('romita', 3),\n",
       " ('mix', 3),\n",
       " ('league', 3),\n",
       " ('like', 3),\n",
       " ('books', 3),\n",
       " ('mirror', 3),\n",
       " ('jesper', 3),\n",
       " ('gold', 3),\n",
       " ('sea', 3),\n",
       " ('shiny', 3),\n",
       " ('beach', 3),\n",
       " ('matte', 3),\n",
       " ('cyberpunk', 3),\n",
       " ('uhd', 3),\n",
       " ('craig', 3),\n",
       " ('her', 3),\n",
       " ('tiny', 3),\n",
       " ('anatomically', 3),\n",
       " ('nose', 3),\n",
       " ('played', 3),\n",
       " ('ejsing', 3),\n",
       " ('victorian', 3),\n",
       " ('correct', 3),\n",
       " ('mullins', 3),\n",
       " ('atmospheric', 3),\n",
       " ('wearing', 3),\n",
       " ('elephant', 3),\n",
       " ('dressed', 3),\n",
       " ('atmosphere', 2),\n",
       " ('cushart', 2),\n",
       " ('made', 2),\n",
       " ('deharme', 2),\n",
       " ('jordan', 2),\n",
       " ('mist', 2),\n",
       " ('steampunk', 2),\n",
       " ('world', 2),\n",
       " ('pelletier', 2),\n",
       " ('django', 2),\n",
       " ('shaped', 2),\n",
       " ('buildings', 2),\n",
       " ('reflections', 2),\n",
       " ('cyborg', 2),\n",
       " ('chinese', 2),\n",
       " ('tower', 2),\n",
       " ('sci', 2),\n",
       " ('exoskeleton', 2),\n",
       " ('sunlight', 2),\n",
       " ('paladin', 2),\n",
       " ('decorated', 2),\n",
       " ('rule', 2),\n",
       " ('souls', 2),\n",
       " ('pool', 2),\n",
       " ('viking', 2),\n",
       " ('section', 2),\n",
       " ('ivory', 2),\n",
       " ('8', 2),\n",
       " ('staring', 2),\n",
       " ('celestial', 2),\n",
       " ('baroque', 2),\n",
       " ('thirds', 2),\n",
       " ('gear', 2),\n",
       " ('biotech', 2),\n",
       " ('middle', 2),\n",
       " ('serious', 2),\n",
       " ('tall', 2),\n",
       " ('gothic', 2),\n",
       " ('karol', 2),\n",
       " ('goddess', 2),\n",
       " ('insanely', 2),\n",
       " ('tran', 2),\n",
       " ('surreal', 2),\n",
       " ('snoeck', 2),\n",
       " ('grimmer', 2),\n",
       " ('bioluminescent', 2),\n",
       " ('headwear', 2),\n",
       " ('glowing', 2),\n",
       " ('fin', 2),\n",
       " ('picture', 2),\n",
       " ('opera', 2),\n",
       " ('statue', 2),\n",
       " ('incredible', 2),\n",
       " ('rendered', 2),\n",
       " ('into', 2),\n",
       " ('cybernetic', 2),\n",
       " ('explosion', 2),\n",
       " ('hypermaximalist', 2),\n",
       " ('paul', 2),\n",
       " ('long', 2),\n",
       " ('riverside', 2),\n",
       " ('survival', 2),\n",
       " ('gorgeous', 2),\n",
       " ('freckles', 2),\n",
       " ('shallow', 2),\n",
       " ('waterfall', 2),\n",
       " ('definition', 2),\n",
       " ('canyon', 2),\n",
       " ('bold', 2),\n",
       " ('natural', 2),\n",
       " ('stars', 2),\n",
       " ('mohrbacher', 2),\n",
       " ('lise', 2),\n",
       " ('bak', 2),\n",
       " ('lovely', 2),\n",
       " ('nlee007', 2),\n",
       " ('looking', 2),\n",
       " ('hat', 2),\n",
       " ('edge', 2),\n",
       " ('escher', 2),\n",
       " ('overcoat', 2),\n",
       " ('happy', 2),\n",
       " ('bisson', 2),\n",
       " ('etheral', 2),\n",
       " ('photorealism', 2),\n",
       " ('peter', 2),\n",
       " ('flowing', 2),\n",
       " ('wet', 2),\n",
       " ('masterpiece', 2),\n",
       " ('winters', 2),\n",
       " ('blonde', 2),\n",
       " ('at', 2),\n",
       " ('darek', 2),\n",
       " ('elaborate', 2),\n",
       " ('space', 2),\n",
       " ('realism', 2),\n",
       " ('roberto', 2),\n",
       " ('k', 2),\n",
       " ('robert', 2),\n",
       " ('tarot', 2),\n",
       " ('batman', 2),\n",
       " ('intrincate', 2),\n",
       " ('walking', 2),\n",
       " ('landscape', 2),\n",
       " ('natalee007', 2),\n",
       " ('laliashvili', 2),\n",
       " ('picturesque', 2),\n",
       " ('head', 2),\n",
       " ('architectural', 2),\n",
       " ('new', 2),\n",
       " ('portal', 2),\n",
       " ('25mm', 2),\n",
       " ('sofie', 2),\n",
       " ('c', 2),\n",
       " ('2', 2),\n",
       " ('visage', 2),\n",
       " ('fluorescence', 2),\n",
       " ('nekro', 2),\n",
       " ('colorful', 2),\n",
       " ('dress', 2),\n",
       " ('1', 2),\n",
       " ('field', 2),\n",
       " ('upperbody', 2),\n",
       " ('warm', 2),\n",
       " ('bob', 2),\n",
       " ('ivan', 2),\n",
       " ('fox', 2),\n",
       " ('fluffy', 2),\n",
       " ('50mm', 2),\n",
       " ('ren', 2),\n",
       " ('depth', 2),\n",
       " ('steps', 2),\n",
       " ('deep', 2),\n",
       " ('james', 2),\n",
       " ('poster', 2),\n",
       " ('krenz', 2),\n",
       " ('j', 2),\n",
       " ('fi', 2),\n",
       " ('rings', 2),\n",
       " ('edouard', 2),\n",
       " ('dropped', 2),\n",
       " ('atompunk', 2),\n",
       " ('garden', 2),\n",
       " ('garment', 2),\n",
       " ('sun', 2),\n",
       " ('adventurer', 2),\n",
       " ('victoria', 2),\n",
       " ('alexis', 2),\n",
       " ('porcelain', 2),\n",
       " ('enchanted', 2),\n",
       " ('zabrocki', 2),\n",
       " ('park', 2),\n",
       " ('diffused', 2),\n",
       " ('outlines', 2),\n",
       " ('gigantic', 2),\n",
       " ('ferri', 2),\n",
       " ('moody', 2),\n",
       " ('features', 2),\n",
       " ('dense', 2),\n",
       " ('feminine', 2),\n",
       " ('complex', 2),\n",
       " ('tracing', 2),\n",
       " ('clothes', 2),\n",
       " ('anger', 2),\n",
       " ('medieval', 2),\n",
       " ('lips', 2),\n",
       " ('f', 2),\n",
       " ('pulp', 2),\n",
       " ('king', 2),\n",
       " ('winter', 2),\n",
       " ('bloodborne', 2),\n",
       " ('obsidian', 2),\n",
       " ('artstyle', 2),\n",
       " ('day', 2),\n",
       " ('inside', 2),\n",
       " ('beta', 2),\n",
       " ('side', 2),\n",
       " ('6', 2),\n",
       " ('clean', 2),\n",
       " ('front', 2),\n",
       " ('infinite', 1),\n",
       " ('pantheon', 1),\n",
       " ('navy', 1),\n",
       " ('photograpy', 1),\n",
       " ('universe', 1),\n",
       " ('tough', 1),\n",
       " ('woodfired', 1),\n",
       " ('brooklyn', 1),\n",
       " ('surfaces', 1),\n",
       " ('chrose', 1),\n",
       " ('cowboy', 1),\n",
       " ('dispersed', 1),\n",
       " ('filthy', 1),\n",
       " ('caravage', 1),\n",
       " ('partially', 1),\n",
       " ('hr', 1),\n",
       " ('gerhartz', 1),\n",
       " ('gal', 1),\n",
       " ('cel', 1),\n",
       " ('chromatic', 1),\n",
       " ('mcconaughey', 1),\n",
       " ('ravenclaw', 1),\n",
       " ('tattoo', 1),\n",
       " ('gemstone', 1),\n",
       " ('summer', 1),\n",
       " ('computational', 1),\n",
       " ('gray', 1),\n",
       " ('cosmonaut', 1),\n",
       " ('model', 1),\n",
       " ('lightning', 1),\n",
       " ('myanmar', 1),\n",
       " ('game', 1),\n",
       " ('brambles', 1),\n",
       " ('movie', 1),\n",
       " ('upwards', 1),\n",
       " ('linen', 1),\n",
       " ('lion', 1),\n",
       " ('oracle', 1),\n",
       " ('fashion', 1),\n",
       " ('fatale', 1),\n",
       " ('hbo', 1),\n",
       " ('mud', 1),\n",
       " ('sleek', 1),\n",
       " ('creating', 1),\n",
       " ('fur', 1),\n",
       " ('evangelion', 1),\n",
       " ('revival', 1),\n",
       " ('f5', 1),\n",
       " ('emblazoned', 1),\n",
       " ('looks', 1),\n",
       " ('mech', 1),\n",
       " ('simonetti', 1),\n",
       " ('yama', 1),\n",
       " ('revealing', 1),\n",
       " ('level', 1),\n",
       " ('seth', 1),\n",
       " ('have', 1),\n",
       " ('grab', 1),\n",
       " ('imposing', 1),\n",
       " ('scifi', 1),\n",
       " ('cliff', 1),\n",
       " ('druid', 1),\n",
       " ('strikingly', 1),\n",
       " ('android', 1),\n",
       " ('human', 1),\n",
       " ('working', 1),\n",
       " ('rhads', 1),\n",
       " ('alkane', 1),\n",
       " ('edwin', 1),\n",
       " ('dust', 1),\n",
       " ('trash', 1),\n",
       " ('infini', 1),\n",
       " ('church', 1),\n",
       " ('redwood', 1),\n",
       " ('mischievous', 1),\n",
       " ('floor', 1),\n",
       " ('inspired', 1),\n",
       " ('key', 1),\n",
       " ('shinkawa', 1),\n",
       " ('print', 1),\n",
       " ('gorge', 1),\n",
       " ('vines', 1),\n",
       " ('bouquet', 1),\n",
       " ('blevins', 1),\n",
       " ('locked', 1),\n",
       " ('blossom', 1),\n",
       " ('form', 1),\n",
       " ('fair', 1),\n",
       " ('nes', 1),\n",
       " ('balaclava', 1),\n",
       " ('zatdaw', 1),\n",
       " ('alps', 1),\n",
       " ('combat', 1),\n",
       " ('geisha', 1),\n",
       " ('hard', 1),\n",
       " ('storytelling', 1),\n",
       " ('psychedelic', 1),\n",
       " ('collection', 1),\n",
       " ('ray', 1),\n",
       " ('empty', 1),\n",
       " ('fantastical', 1),\n",
       " ('frame', 1),\n",
       " ('large', 1),\n",
       " ('yoshida', 1),\n",
       " ('100', 1),\n",
       " ('granite', 1),\n",
       " ('weilding', 1),\n",
       " ('monks', 1),\n",
       " ('sorceress', 1),\n",
       " ('era', 1),\n",
       " ('among', 1),\n",
       " ('queen', 1),\n",
       " ('staff', 1),\n",
       " ('takahashi', 1),\n",
       " ('indoislamic', 1),\n",
       " ('ultrahd', 1),\n",
       " ('maiden', 1),\n",
       " ('dystopian', 1),\n",
       " ('chiaroscuro', 1),\n",
       " ('roof', 1),\n",
       " ('de', 1),\n",
       " ('southamerican', 1),\n",
       " ('wondrous', 1),\n",
       " ('shining', 1),\n",
       " ('vortex', 1),\n",
       " ('colored', 1),\n",
       " ('ponytail', 1),\n",
       " ('dappling', 1),\n",
       " ('tomyoji', 1),\n",
       " ('clouds', 1),\n",
       " ('shape', 1),\n",
       " ('cloud', 1),\n",
       " ('christ', 1),\n",
       " ('grey', 1),\n",
       " ('map', 1),\n",
       " ('creepy', 1),\n",
       " ('octrender', 1),\n",
       " ('scarfs', 1),\n",
       " ('horse', 1),\n",
       " ('nuclear', 1),\n",
       " ('dan', 1),\n",
       " ('akihiko', 1),\n",
       " ('hypnos', 1),\n",
       " ('sleep', 1),\n",
       " ('fineart', 1),\n",
       " ('lens', 1),\n",
       " ('itinerary', 1),\n",
       " ('towards', 1),\n",
       " ('irezumi', 1),\n",
       " ('spirals', 1),\n",
       " ('vase', 1),\n",
       " ('insane', 1),\n",
       " ('fractals', 1),\n",
       " ('samurai', 1),\n",
       " ('flying', 1),\n",
       " ('incredibly', 1),\n",
       " ('mirrors', 1),\n",
       " ('mystery', 1),\n",
       " ('noise', 1),\n",
       " ('neon', 1),\n",
       " ('apocalyptic', 1),\n",
       " ('after', 1),\n",
       " ('fashionable', 1),\n",
       " ('sublime', 1),\n",
       " ('archangel', 1),\n",
       " ('katsuhiro', 1),\n",
       " ('sweater', 1),\n",
       " ('plumbing', 1),\n",
       " ('pronounced', 1),\n",
       " ('rabbit', 1),\n",
       " ('poses', 1),\n",
       " ('show', 1),\n",
       " ('bird', 1),\n",
       " ('kodak', 1),\n",
       " ('brush', 1),\n",
       " ('fierce', 1),\n",
       " ('environment', 1),\n",
       " ('wings', 1),\n",
       " ('vray', 1),\n",
       " ('uplight', 1),\n",
       " ('doors', 1),\n",
       " ('robe', 1),\n",
       " ('entire', 1),\n",
       " ('climbing', 1),\n",
       " ('aperture', 1),\n",
       " ('region', 1),\n",
       " ('jesus', 1),\n",
       " ('landscapes', 1),\n",
       " ('opal', 1),\n",
       " ('x', 1),\n",
       " ('hairstyle', 1),\n",
       " ('sword', 1),\n",
       " ('infite', 1),\n",
       " ('all', 1),\n",
       " ('hardsurface', 1),\n",
       " ('cam', 1),\n",
       " ('fusion', 1),\n",
       " ('khaki', 1),\n",
       " ('birds', 1),\n",
       " ('mcbess', 1),\n",
       " ('shell', 1),\n",
       " ('earings', 1),\n",
       " ('wire', 1),\n",
       " ('wonder', 1),\n",
       " ('suit', 1),\n",
       " ('color', 1),\n",
       " ('fallen', 1),\n",
       " ('glazed', 1),\n",
       " ('tor', 1),\n",
       " ('pajama', 1),\n",
       " ('overcast', 1),\n",
       " ('expression', 1),\n",
       " ('knitted', 1),\n",
       " ('alley', 1),\n",
       " ('palette', 1),\n",
       " ('gery', 1),\n",
       " ('cgi', 1),\n",
       " ('neil', 1),\n",
       " ('emotional', 1),\n",
       " ('rainbow', 1),\n",
       " ('buttons', 1),\n",
       " ('biomechanical', 1),\n",
       " ('ocean', 1),\n",
       " ('renderer', 1),\n",
       " ('ar', 1),\n",
       " ('fetishistic', 1),\n",
       " ('telephoto', 1),\n",
       " ('thick', 1),\n",
       " ('tiled', 1),\n",
       " ('trail', 1),\n",
       " ('zero', 1),\n",
       " ('aberration', 1),\n",
       " ('maze', 1),\n",
       " ('beautifully', 1),\n",
       " ('babe', 1),\n",
       " ('path', 1),\n",
       " ('sideswept', 1),\n",
       " ('swirles', 1),\n",
       " ('spell', 1),\n",
       " ('appearing', 1),\n",
       " ('matthew', 1),\n",
       " ('spooky', 1),\n",
       " ('michael', 1),\n",
       " ('slaby', 1),\n",
       " ('charismatic', 1),\n",
       " ('collapsed', 1),\n",
       " ('glasses', 1),\n",
       " ('dirt', 1),\n",
       " ('threatening', 1),\n",
       " ('sweet', 1),\n",
       " ('figurative', 1),\n",
       " ('anno', 1),\n",
       " ('dresses', 1),\n",
       " ('trip', 1),\n",
       " ('some', 1),\n",
       " ('deco', 1),\n",
       " ('alien', 1),\n",
       " ('mythology', 1),\n",
       " ('crazy', 1),\n",
       " ('aerial', 1),\n",
       " ('interstellar', 1),\n",
       " ('sfx', 1),\n",
       " ('jewels', 1),\n",
       " ('carbon', 1),\n",
       " ('grahame', 1),\n",
       " ('mexican', 1),\n",
       " ('river', 1),\n",
       " ('asian', 1),\n",
       " ('simon', 1),\n",
       " ('kassan', 1),\n",
       " ('carved', 1),\n",
       " ('yellow', 1),\n",
       " ('62', 1),\n",
       " ('centre', 1),\n",
       " ('dawn', 1),\n",
       " ('labeled', 1),\n",
       " ('waxwing', 1),\n",
       " ('gadot', 1),\n",
       " ('enhanced', 1),\n",
       " ('ragged', 1),\n",
       " ('gemstones', 1),\n",
       " ('war', 1),\n",
       " ('complementary', 1),\n",
       " ('howl', 1),\n",
       " ('shoden', 1),\n",
       " ('eden', 1),\n",
       " ('steam', 1),\n",
       " ('wave', 1),\n",
       " ('airy', 1),\n",
       " ('cg', 1),\n",
       " ('emerging', 1),\n",
       " ('engineer', 1),\n",
       " ('off', 1),\n",
       " ('ratajkowski', 1),\n",
       " ('raven', 1),\n",
       " ('bohemian', 1),\n",
       " ('confident', 1),\n",
       " ('theatre', 1),\n",
       " ('smirking', 1),\n",
       " ('55mm', 1),\n",
       " ('earth', 1),\n",
       " ('chief', 1),\n",
       " ('ghostly', 1),\n",
       " ('corben', 1),\n",
       " ('amazing', 1),\n",
       " ('cherry', 1),\n",
       " ('created', 1),\n",
       " ('warcraft', 1),\n",
       " ('tank', 1),\n",
       " ('tribal', 1),\n",
       " ('translucent', 1),\n",
       " ('years', 1),\n",
       " ('smith', 1),\n",
       " ('gradient', 1),\n",
       " ('brandenburger', 1),\n",
       " ('spring', 1),\n",
       " ('horizon', 1),\n",
       " ('h', 1),\n",
       " ('gallery', 1),\n",
       " ('glitch', 1),\n",
       " ('bottle', 1),\n",
       " ('eye', 1),\n",
       " ('ryden', 1),\n",
       " ('facepaint', 1),\n",
       " ('raiden', 1),\n",
       " ('upton', 1),\n",
       " ('captain', 1),\n",
       " ('oregon', 1),\n",
       " ('sly', 1),\n",
       " ('heaven', 1),\n",
       " ('who', 1),\n",
       " ('dof', 1),\n",
       " ('was', 1),\n",
       " ('swashbucking', 1),\n",
       " ('muscular', 1),\n",
       " ('symbol', 1),\n",
       " ('effect', 1),\n",
       " ('moonlight', 1),\n",
       " ('fearsome', 1),\n",
       " ('extra', 1),\n",
       " ('athletic', 1),\n",
       " ('fading', 1),\n",
       " ('star', 1),\n",
       " ('500px', 1),\n",
       " ('ghost', 1),\n",
       " ('lively', 1),\n",
       " ('mouse', 1),\n",
       " ('zebra', 1),\n",
       " ('zhong', 1),\n",
       " ('cobra', 1),\n",
       " ('femme', 1),\n",
       " ('theater', 1),\n",
       " ('smouldering', 1),\n",
       " ('intricated', 1),\n",
       " ('mirren', 1),\n",
       " ('19th', 1),\n",
       " ('mumford', 1),\n",
       " ('embellishments', 1),\n",
       " ('uniform', 1),\n",
       " ('damned', 1),\n",
       " ('cloth', 1),\n",
       " ('wooden', 1),\n",
       " ('bussiere', 1),\n",
       " ('wars', 1),\n",
       " ('deitaled', 1),\n",
       " ('stands', 1),\n",
       " ('pirates', 1),\n",
       " ('delicate', 1),\n",
       " ('intricately', 1),\n",
       " ('piglet', 1),\n",
       " ('daniel', 1),\n",
       " ('air', 1),\n",
       " ('silent', 1),\n",
       " ('main', 1),\n",
       " ('make', 1),\n",
       " ('surrealism', 1),\n",
       " ('emily', 1),\n",
       " ('fenghua', 1),\n",
       " ('focused', 1),\n",
       " ('rainbows', 1),\n",
       " ('spiraling', 1),\n",
       " ('quality', 1),\n",
       " ('wolf', 1),\n",
       " ('unit', 1),\n",
       " ('mignola', 1),\n",
       " ('curvaceous', 1),\n",
       " ('ornamented', 1),\n",
       " ('guay', 1),\n",
       " ('there', 1),\n",
       " ('fate', 1),\n",
       " ('jewish', 1),\n",
       " ('designs', 1),\n",
       " ('sigil', 1),\n",
       " ('evil', 1),\n",
       " ('cables', 1),\n",
       " ('butterfly', 1),\n",
       " ('york', 1),\n",
       " ('moving', 1),\n",
       " ('reylia', 1),\n",
       " ('scape', 1),\n",
       " ('interior', 1),\n",
       " ('striped', 1),\n",
       " ('stargate', 1),\n",
       " ('contained', 1),\n",
       " ('around', 1),\n",
       " ('di', 1),\n",
       " ('egyptian', 1),\n",
       " ('rose', 1),\n",
       " ('laica', 1),\n",
       " ('ggulbae', 1),\n",
       " ('creek', 1),\n",
       " ('banksy', 1),\n",
       " ('hokusai', 1),\n",
       " ('creatures', 1),\n",
       " ('hobbit', 1),\n",
       " ('horror', 1),\n",
       " ('wreathed', 1),\n",
       " ('extreme', 1),\n",
       " ('rain', 1),\n",
       " ('ever', 1),\n",
       " ('tumor', 1),\n",
       " ('sakimichan', 1),\n",
       " ('anastasia', 1),\n",
       " ('taunting', 1),\n",
       " ('otomo', 1),\n",
       " ('core', 1),\n",
       " ('rage', 1),\n",
       " ('caribbean', 1),\n",
       " ('visible', 1),\n",
       " ('smile', 1),\n",
       " ('deadly', 1),\n",
       " ('planets', 1),\n",
       " ('dreams', 1),\n",
       " ('hideaki', 1),\n",
       " ('digitally', 1),\n",
       " ('curves', 1),\n",
       " ('challenging', 1),\n",
       " ('torn', 1),\n",
       " ('platforms', 1),\n",
       " ('down', 1),\n",
       " ('sapphire', 1),\n",
       " ('perfection', 1),\n",
       " ('vincent', 1),\n",
       " ('roses', 1),\n",
       " ('chopra', 1),\n",
       " ('endless', 1),\n",
       " ('romantic', 1),\n",
       " ('leyendecker', 1),\n",
       " ('robot', 1),\n",
       " ('villaincore', 1),\n",
       " ('tied', 1),\n",
       " ('marc', 1),\n",
       " ('txaa', 1),\n",
       " ('muted', 1),\n",
       " ('stoic', 1),\n",
       " ('guweiz', 1),\n",
       " ('pin', 1),\n",
       " ('center', 1),\n",
       " ('huge', 1),\n",
       " ('milkyway', 1),\n",
       " ('shield', 1),\n",
       " ('geometrical', 1),\n",
       " ('cutest', 1),\n",
       " ('exposed', 1),\n",
       " ('waterplants', 1),\n",
       " ('luppova', 1),\n",
       " ('gas', 1),\n",
       " ('fire', 1),\n",
       " ('military', 1),\n",
       " ('goat', 1),\n",
       " ('how', 1),\n",
       " ('painter', 1),\n",
       " ('royal', 1),\n",
       " ('carving', 1),\n",
       " ('photos', 1),\n",
       " ('backlight', 1),\n",
       " ('gundam', 1),\n",
       " ('ice', 1),\n",
       " ('layers', 1),\n",
       " ('muppet', 1),\n",
       " ('glamour', 1),\n",
       " ('theme', 1),\n",
       " ('nadar', 1),\n",
       " ('giant', 1),\n",
       " ('pbr', 1),\n",
       " ('tuxedo', 1),\n",
       " ('spirited', 1),\n",
       " ('scary', 1),\n",
       " ('relentless', 1),\n",
       " ('piercing', 1),\n",
       " ('kombat', 1),\n",
       " ('reflection', 1),\n",
       " ('candle', 1),\n",
       " ('witch', 1),\n",
       " ('jar', 1),\n",
       " ('facula', 1),\n",
       " ('ultimate', 1),\n",
       " ('latex', 1),\n",
       " ('child', 1),\n",
       " ('alyssa', 1),\n",
       " ('marble', 1),\n",
       " ('porcelan', 1),\n",
       " ('gaston', 1),\n",
       " ('particle', 1),\n",
       " ('rumiko', 1),\n",
       " ('through', 1),\n",
       " ('davinci', 1),\n",
       " ('highquality', 1),\n",
       " ('symmetric', 1),\n",
       " ('hyperdetailed', 1),\n",
       " ('crumb', 1),\n",
       " ('between', 1),\n",
       " ('void', 1),\n",
       " ('nebula', 1),\n",
       " ('morbide', 1),\n",
       " ('little', 1),\n",
       " ('mark', 1),\n",
       " ('sim', 1),\n",
       " ('blurred', 1),\n",
       " ('gloomy', 1),\n",
       " ('garbage', 1),\n",
       " ('its', 1),\n",
       " ('standing', 1),\n",
       " ('unchained', 1),\n",
       " ('zoom', 1),\n",
       " ('photographed', 1),\n",
       " ('dissolving', 1),\n",
       " ('shire', 1),\n",
       " ('tolkien', 1),\n",
       " ('korpi', 1),\n",
       " ('circular', 1),\n",
       " ('dragon', 1),\n",
       " ('rubber', 1),\n",
       " ('midnight', 1),\n",
       " ('tatooes', 1),\n",
       " ('wind', 1),\n",
       " ('camouflage', 1),\n",
       " ('topaz', 1),\n",
       " ('fern', 1),\n",
       " ('spikes', 1),\n",
       " ('fractal', 1),\n",
       " ('guard', 1),\n",
       " ('outdoors', 1),\n",
       " ('emu', 1),\n",
       " ('jacarandas', 1),\n",
       " ('3', 1),\n",
       " ('primeval', 1),\n",
       " ('garrulus', 1),\n",
       " ('onesies', 1),\n",
       " ('two', 1),\n",
       " ('graphic', 1),\n",
       " ('dynamic', 1),\n",
       " ('stylize', 1),\n",
       " ('volume', 1),\n",
       " ('mushrooms', 1),\n",
       " ('building', 1),\n",
       " ('bioluminescence', 1),\n",
       " ('great', 1),\n",
       " ('southeast', 1),\n",
       " ('kneeling', 1),\n",
       " ('pastel', 1),\n",
       " ('eerie', 1),\n",
       " ('handsome', 1),\n",
       " ('vibrant', 1),\n",
       " ('01', 1),\n",
       " ('equipment', 1),\n",
       " ...]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# words frequency statistics\n",
    "import re\n",
    "\n",
    "file = open('./corpus/templates.txt', 'r')\n",
    "templates_txt = file.read()\n",
    "\n",
    "words = re.findall(r'\\w+', templates_txt)\n",
    "words = [w.lower() for w in words]\n",
    "vocabulary = set(words)\n",
    "frequency = [(word, words.count(word)) for word in vocabulary]\n",
    "frequency = sorted(frequency, key=lambda x: -x[1])\n",
    "\n",
    "frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_words = [pair[0] for pair in frequency if pair[1] >= 5]\n",
    "print('\\n'.join(hf_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read file names in a folder\n",
    "import os\n",
    "\n",
    "\n",
    "with open('corpus/templates-A.txt', 'a') as output:\n",
    "\troot, dir, files = next(os.walk(r'~\\512x1024\\A'))\n",
    "\tlines = [name.replace('.png', '') for name in files]\n",
    "\toutput.write('\\n'.join(lines) + '\\n')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a7d38e74bfe13cb98d9c208d510fb3d2ada38d33ea651cd79db8b0db1ba5c65d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}