Code and data ready

Author: allenjack

dataset/Amazon.py

@@ -0,0 +1,134 @@
+from dataset.Dataset import DataSet
+
+
+# Amazon review dataset
+class Electronics(DataSet):
+    def __init__(self):
+        self.dir_path = './dataset/data/amazon/Electronics/'
+        self.user_record_file = 'Electronics_user_records.pkl'
+        self.user_mapping_file = 'Electronics_user_mapping.pkl'
+        self.item_mapping_file = 'Electronics_item_mapping.pkl'
+        self.item_content_file = 'word_counts.txt'
+        self.item_relation_file = 'item_relation.pkl'
+
+        # data structures used in the model
+        self.num_users = 37204
+        self.num_items = 13881
+        self.vocab_size = 10104
+
+        self.user_records = None
+        self.user_mapping = None
+        self.item_mapping = None
+
+    def generate_dataset(self, seed=0):
+        user_records = self.load_pickle(self.dir_path + self.user_record_file)
+        user_mapping = self.load_pickle(self.dir_path + self.user_mapping_file)
+        item_mapping = self.load_pickle(self.dir_path + self.item_mapping_file)
+
+        self.num_users = len(user_mapping)
+        self.num_items = len(item_mapping)
+
+        inner_data_records, user_inverse_mapping, item_inverse_mapping = \
+            self.convert_to_inner_index(user_records, user_mapping, item_mapping)
+
+        train_set, test_set = self.split_data_randomly(inner_data_records, seed)
+
+        train_matrix = self.generate_rating_matrix(train_set, self.num_users, self.num_items)
+        # train_matrix = self.fill_zero_col(train_matrix)
+        item_content_matrix = self.load_item_content(self.dir_path + self.item_content_file, self.vocab_size)
+        item_relation_matrix = self.load_pickle(self.dir_path + self.item_relation_file)
+
+        return train_matrix, train_set, test_set, item_content_matrix, item_relation_matrix
+
+
+class Books(DataSet):
+    def __init__(self):
+        self.dir_path = './dataset/data/amazon/Books/'
+        self.user_record_file = 'Books_user_records.pkl'
+        self.user_mapping_file = 'Books_user_mapping.pkl'
+        self.item_mapping_file = 'Books_item_mapping.pkl'
+        self.item_content_file = 'word_counts.txt'
+        self.item_relation_file = 'item_relation.pkl'
+        self.item_word_seq_file = 'review_word_sequence.pkl'
+
+        # data structures used in the model
+        self.num_users = 65476
+        self.num_items = 41264
+        self.vocab_size = 27584
+
+        self.user_records = None
+        self.user_mapping = None
+        self.item_mapping = None
+
+    def generate_dataset(self, seed=0):
+        user_records = self.load_pickle(self.dir_path + self.user_record_file)
+        user_mapping = self.load_pickle(self.dir_path + self.user_mapping_file)
+        item_mapping = self.load_pickle(self.dir_path + self.item_mapping_file)
+        word_seq = self.load_pickle(self.dir_path + self.item_word_seq_file)
+
+        self.num_users = len(user_mapping)
+        self.num_items = len(item_mapping)
+
+        inner_data_records, user_inverse_mapping, item_inverse_mapping = \
+            self.convert_to_inner_index(user_records, user_mapping, item_mapping)
+
+        train_set, test_set = self.split_data_randomly(inner_data_records, seed)
+
+        train_matrix = self.generate_rating_matrix(train_set, self.num_users, self.num_items)
+        item_content_matrix = self.load_item_content(self.dir_path + self.item_content_file, self.vocab_size)
+        item_relation_matrix = self.load_pickle(self.dir_path + self.item_relation_file)
+
+        return train_matrix, train_set, test_set, item_content_matrix, item_relation_matrix, word_seq
+
+
+class CDs(DataSet):
+    def __init__(self):
+        self.dir_path = './dataset/data/amazon/CDs/'
+        self.user_record_file = 'CDs_user_records.pkl'
+        self.user_mapping_file = 'CDs_user_mapping.pkl'
+        self.item_mapping_file = 'CDs_item_mapping.pkl'
+        self.item_content_file = 'word_counts.txt'
+        self.item_relation_file = 'item_relation.pkl'
+        self.item_word_seq_file = 'review_word_sequence.pkl'
+
+        # data structures used in the model
+        self.num_users = 24934
+        self.num_items = 24634
+        self.vocab_size = 24341
+
+        self.user_records = None
+        self.user_mapping = None
+        self.item_mapping = None
+
+    def generate_dataset(self, seed=0):
+        user_records = self.load_pickle(self.dir_path + self.user_record_file)
+        user_mapping = self.load_pickle(self.dir_path + self.user_mapping_file)
+        item_mapping = self.load_pickle(self.dir_path + self.item_mapping_file)
+        word_seq = self.load_pickle(self.dir_path + self.item_word_seq_file)
+
+        self.num_users = len(user_mapping)
+        self.num_items = len(item_mapping)
+
+        inner_data_records, user_inverse_mapping, item_inverse_mapping = \
+            self.convert_to_inner_index(user_records, user_mapping, item_mapping)
+
+        train_set, test_set = self.split_data_randomly(inner_data_records, seed)
+
+        train_matrix = self.generate_rating_matrix(train_set, self.num_users, self.num_items)
+        item_content_matrix = self.load_item_content(self.dir_path + self.item_content_file, self.vocab_size)
+        item_relation_matrix = self.load_pickle(self.dir_path + self.item_relation_file)
+
+        return train_matrix, train_set, test_set, item_content_matrix, item_relation_matrix, word_seq
+
+
+if __name__ == '__main__':
+    data_set = CDs()
+    train_matrix, train_set, test_set, item_content_matrix, item_relation_matrix, word_seq = data_set.generate_dataset()
+    print(word_seq[-1])
+    max_len = 0
+    for word_list in word_seq:
+        max_len = max(len(word_list), max_len)
+    print(max_len)
+    for i in range(item_content_matrix.shape[0]):
+        if item_content_matrix.getrow(i).getnnz() == 0:
+            print(i)

dataset/Dataset.py

@@ -0,0 +1,105 @@
+import pickle
+import math
+
+import numpy as np
+from scipy.sparse import csr_matrix
+from sklearn.model_selection import train_test_split
+
+
+class DataSet(object):
+    def load_pickle(self, name):
+        with open(name, 'rb') as f:
+            return pickle.load(f, encoding='latin1')
+
+    def generate_inverse_mapping(self, data_list):
+        inverse_mapping = dict()
+        for inner_id, true_id in enumerate(data_list):
+            inverse_mapping[true_id] = inner_id
+        return inverse_mapping
+
+    def convert_to_inner_index(self, user_records, user_mapping, item_mapping):
+        inner_user_records = []
+        user_inverse_mapping = self.generate_inverse_mapping(user_mapping)
+        item_inverse_mapping = self.generate_inverse_mapping(item_mapping)
+
+        for user_id in range(len(user_mapping)):
+            real_user_id = user_mapping[user_id]
+            item_list = list(user_records[real_user_id])
+            for index, real_item_id in enumerate(item_list):
+                item_list[index] = item_inverse_mapping[real_item_id]
+            inner_user_records.append(item_list)
+
+        return inner_user_records, user_inverse_mapping, item_inverse_mapping
+
+    def split_data_randomly(self, user_records, seed=0):
+        # randomly hold part of the data as the test set
+        test_ratio = 0.2
+        train_set = []
+        test_set = []
+        for user_id, item_list in enumerate(user_records):
+            tmp_train_sample, tmp_test_sample = train_test_split(item_list, test_size=test_ratio, random_state=seed)
+
+            train_sample = []
+            for place in item_list:
+                if place not in tmp_test_sample:
+                    train_sample.append(place)
+
+            test_sample = []
+            for place in tmp_test_sample:
+                test_sample.append(place)
+
+            train_set.append(train_sample)
+            test_set.append(test_sample)
+        return train_set, test_set
+
+    def split_data_sequentially(self, user_records):
+        test_radio = 0.2
+        train_set = []
+        test_set = []
+
+        for item_list in user_records:
+            len_list = len(item_list)
+            num_test_samples = int(math.ceil(len_list * test_radio))
+            train_sample = []
+            test_sample = []
+            for i in range(len_list - num_test_samples, len_list):
+                test_sample.append(item_list[i])
+
+            for place in item_list:
+                if place not in set(test_sample):
+                    train_sample.append(place)
+
+            train_set.append(train_sample)
+            test_set.append(test_sample)
+
+        return train_set, test_set
+
+    def generate_rating_matrix(self, train_set, num_users, num_items):
+        # three lists are used to construct sparse matrix
+        row = []
+        col = []
+        data = []
+        for user_id, article_list in enumerate(train_set):
+            for article in article_list:
+                row.append(user_id)
+                col.append(article)
+                data.append(1)
+
+        row = np.array(row)
+        col = np.array(col)
+        data = np.array(data)
+        rating_matrix = csr_matrix((data, (row, col)), shape=(num_users, num_items))
+
+        return rating_matrix
+
+    def load_item_content(self, f_in, D=8000):
+        fp = open(f_in)
+        lines = fp.readlines()
+        X = np.zeros((len(lines), D))
+        for i, line in enumerate(lines):
+            strs = line.strip().split(' ')[2:]
+            for strr in strs:
+                segs = strr.split(':')
+                X[i, int(segs[0])] = float(segs[1])
+
+        return csr_matrix(X)