Refactor/roster

big_tests/rebar.config

@@ -10,7 +10,7 @@
 {deps, [
         {lager, "3.9.2"},
         {base16, "2.0.1"},
-        {exml, "3.4.0", {pkg, hexml}},
+        {exml, "3.4.1", {pkg, hexml}},
         {bbmustache, "1.12.2"},
         {jiffy, "1.1.1"},
         {proper, "1.4.0"},
@@ -21,5 +21,5 @@
         {csv, "3.0.3", {pkg, csve}},
         {amqp_client, "3.12.6"},
         {esip, "1.0.48"},
-        {jid, "2.0.0", {pkg, mongoose_jid}}
+        {jid, "2.1.0", {pkg, mongoose_jid}}
 ]}.

big_tests/rebar.lock

@@ -10,14 +10,14 @@
  {<<"csv">>,{pkg,<<"csve">>,<<"3.0.3">>},0},
  {<<"escalus">>,{pkg,<<"escalus">>,<<"4.2.11">>},0},
  {<<"esip">>,{pkg,<<"esip">>,<<"1.0.48">>},0},
- {<<"exml">>,{pkg,<<"hexml">>,<<"3.4.0">>},0},
+ {<<"exml">>,{pkg,<<"hexml">>,<<"3.4.1">>},0},
  {<<"fast_pbkdf2">>,{pkg,<<"fast_pbkdf2">>,<<"1.0.3">>},2},
  {<<"fast_scram">>,{pkg,<<"fast_scram">>,<<"0.5.0">>},1},
  {<<"fast_tls">>,{pkg,<<"fast_tls">>,<<"1.1.16">>},1},
  {<<"fusco">>,{pkg,<<"fusco">>,<<"0.1.1">>},0},
  {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
  {<<"gun">>,{pkg,<<"gun">>,<<"2.0.0-rc.2">>},0},
- {<<"jid">>,{pkg,<<"mongoose_jid">>,<<"2.0.0">>},0},
+ {<<"jid">>,{pkg,<<"mongoose_jid">>,<<"2.1.0">>},0},
  {<<"jiffy">>,{pkg,<<"jiffy">>,<<"1.1.1">>},0},
  {<<"lager">>,{pkg,<<"lager">>,<<"3.9.2">>},0},
  {<<"meck">>,{pkg,<<"meck">>,<<"0.9.2">>},1},
@@ -27,7 +27,7 @@
  {<<"rabbit_common">>,{pkg,<<"rabbit_common">>,<<"3.12.6">>},1},
  {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},1},
  {<<"recon">>,{pkg,<<"recon">>,<<"2.5.3">>},2},
- {<<"stringprep">>,{pkg,<<"stringprep">>,<<"1.0.27">>},1},
+ {<<"stringprep">>,{pkg,<<"stringprep">>,<<"1.0.29">>},1},
  {<<"stun">>,{pkg,<<"stun">>,<<"1.2.6">>},1},
  {<<"thoas">>,{pkg,<<"thoas">>,<<"1.0.0">>},2},
  {<<"uuid">>,{pkg,<<"uuid_erl">>,<<"2.0.7">>},1},
@@ -43,14 +43,14 @@
  {<<"csv">>, <<"69E7D9B3FDC72016644368762C6A3E6CBFEB85BCCADBF1BD99AB6C827E360E04">>},
  {<<"escalus">>, <<"FBEC4F874AACCFD5B493EE9CFE39F7FE9DDADFCB5054DFD83B8965B64ED43497">>},
  {<<"esip">>, <<"3B3B3AFC798BE9458517D4FD2730674322368E54C2C1211AA630327354946D1B">>},
- {<<"exml">>, <<"18F9B02107E14F6CCDEFBDB7FEB0706C56AD3AFD0DD524BDD1F0ECD98746B9BC">>},
+ {<<"exml">>, <<"9581FE6512D9772C61BBE611CD4A8E5BB90B4D4481275325EC520F7A931A9393">>},
  {<<"fast_pbkdf2">>, <<"4F09D6C6C20DBEE1970E0A6AE91432E1B7731F88426C671D083BAC31FFA1FDAD">>},
  {<<"fast_scram">>, <<"BD0B946911B07D36EC22AD950FF22F77F27B94E4E24452AF966597C6D8CB5E7F">>},
  {<<"fast_tls">>, <<"85FA7F3112EA4FF5CCB4F3ABADC130A8C855AD74EB00869487399CB0C322D208">>},
  {<<"fusco">>, <<"3DD6A90151DFEF30EA1937CC44E9A59177C0094918388D9BCAA2F2DC5E2AE4AA">>},
  {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
  {<<"gun">>, <<"7C489A32DEDCCB77B6E82D1F3C5A7DADFBFA004EC14E322CDB5E579C438632D2">>},
- {<<"jid">>, <<"0D0FD1130EAD05DF672D1C9E0405D3000F0C87A7D446577755CF5EB173104FE9">>},
+ {<<"jid">>, <<"7798E7443124BF93A58E56794D6F28CDF126A1AA7B318663E762287C9CFE9397">>},
  {<<"jiffy">>, <<"ACA10F47AA91697BF24AB9582C74E00E8E95474C7EF9F76D4F1A338D0F5DE21B">>},
  {<<"lager">>, <<"4CAB289120EB24964E3886BD22323CB5FEFE4510C076992A23AD18CF85413D8C">>},
  {<<"meck">>, <<"85CCBAB053F1DB86C7CA240E9FC718170EE5BDA03810A6292B5306BF31BAE5F5">>},
@@ -60,7 +60,7 @@
  {<<"rabbit_common">>, <<"FEBD37E11483F94B614CD636C1EDBCE1099FF64866598F75D0A624D3A60437C6">>},
  {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>},
  {<<"recon">>, <<"739107B9050EA683C30E96DE050BC59248FD27EC147696F79A8797FF9FA17153">>},
- {<<"stringprep">>, <<"02808C7024BC6285CA6A8A67E7ADDFC16F35DDA55551A582C5181D8EA960E890">>},
+ {<<"stringprep">>, <<"02F23E8C3A219A3DFE40A22E908BECE3A2F68AF0FF599EA8A7B714ECB21E62EE">>},
  {<<"stun">>, <<"5D1978D340EA20EFB28BC1E58779A3A1D64568C66168DB4D20692E76CE813D5E">>},
  {<<"thoas">>, <<"567C03902920827A18A89F05B79A37B5BF93553154B883E0131801600CF02CE0">>},
  {<<"uuid">>, <<"B2078D2CC814F53AFA52D36C91E08962C7E7373585C623F4C0EA6DFB04B2AF94">>},
@@ -75,14 +75,14 @@
  {<<"csv">>, <<"741D1A55AABADAA3E0FE13051050101A73E90C4570B9F9403A939D9546813521">>},
  {<<"escalus">>, <<"4E1262E6B1EEF834FE421FA1D3BBE69EF2028A8B88140DC397703B31B904C6D0">>},
  {<<"esip">>, <<"02B9FC6E071415CBC62105F5115AEB68D11184BDAD3960DA7B62EA3E99E7FCCF">>},
- {<<"exml">>, <<"5F73224B784D45C743A949FC5AF4BB441B6B546B3D9883C0441D45401505ADD2">>},
+ {<<"exml">>, <<"D8E7894E2544402B4986EEB2443C15B51B14F686266F091DBF2777D1D99A2FA2">>},
  {<<"fast_pbkdf2">>, <<"2900431E2E6402F23A92754448BBD949DA366BC9C984FDC791DDCFCC41042434">>},
  {<<"fast_scram">>, <<"D45B746B8D17C0F73A94CBDB526E115A4AEDFECB4D42CA935BCB1195D783B4E1">>},
  {<<"fast_tls">>, <<"AA08CCA89B4044E74F1F12E399817D8BEAEAE3EE006C98A893C0BFB1D81FBA51">>},
  {<<"fusco">>, <<"6343551BD1E824F2A6CA85E1158C5B37C320FD449FBFEC7450A73F192AAF9022">>},
  {<<"goldrush">>, <<"99CB4128CFFCB3227581E5D4D803D5413FA643F4EB96523F77D9E6937D994CEB">>},
  {<<"gun">>, <<"6B9D1EAE146410D727140DBF8B404B9631302ECC2066D1D12F22097AD7D254FC">>},
- {<<"jid">>, <<"7BAE8403552D5BCECD1CD2A848445B5FC3533F5EEB355173A0531A1A1AA8E008">>},
+ {<<"jid">>, <<"C2F5B32C9CE8B7B9F321E4DCFE12380CD6CB93CE2208312570D65A06CBA24837">>},
  {<<"jiffy">>, <<"62E1F0581C3C19C33A725C781DFA88410D8BFF1BBAFC3885A2552286B4785C4C">>},
  {<<"lager">>, <<"7F904D9E87A8CB7E66156ED31768D1C8E26EBA1D54F4BC85B1AA4AC1F6340C28">>},
  {<<"meck">>, <<"81344F561357DC40A8344AFA53767C32669153355B626EA9FCBC8DA6B3045826">>},
@@ -92,7 +92,7 @@
  {<<"rabbit_common">>, <<"D85282C8C9BE456B42AA4B265EDE68D176CA8A28DFCF1D521BE19267167C0DC3">>},
  {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>},
  {<<"recon">>, <<"6C6683F46FD4A1DFD98404B9F78DCABC7FCD8826613A89DCB984727A8C3099D7">>},
- {<<"stringprep">>, <<"A5967B1144CA8002A58A03D16DD109FBD0BCDB82616CEAD2F983944314AF6A00">>},
+ {<<"stringprep">>, <<"928EBA304C3006EB1512110EBD7B87DB163B00859A09375A1E4466152C6C462A">>},
  {<<"stun">>, <<"21AED098457E5099E925129459590592E001C470CF7503E5614A7A6B688FF146">>},
  {<<"thoas">>, <<"FC763185B932ECB32A554FB735EE03C3B6B1B31366077A2427D2A97F3BD26735">>},
  {<<"uuid">>, <<"4E4C5CA3461DC47C5E157ED42AA3981A053B7A186792AF972A27B14A9489324E">>},

doc/migrations/6.2.0_6.2.1.md

@@ -0,0 +1,7 @@
+## Presences
+
+`mod_presences` was internally refactored to modernise and improve the performance of the code, but as a side-effect, code for [XEP-0018](https://xmpp.org/extensions/xep-0018.html) was removed. Note that this XEP was not advertised and as a matter of fact was deprecated already in 2003, so if your client was depending on it, it is high time to update.
+
+## Roster
+
+`mod_roster` was internally refactored to modernise and improve the performance of the code, but as a side-effect, some database migrations need to be carried. See the migrations for Postgres, MySQL and MSSQL in the [`priv/migrations`](https://github.com/esl/MongooseIM/tree/master/priv/migrations) directory.

include/mod_roster.hrl

@@ -18,15 +18,15 @@
 %%%
 %%%----------------------------------------------------------------------
 
--record(roster, {usj :: {jid:luser(), jid:lserver(), jid:ljid()},
-                 us :: {jid:luser(), jid:lserver()},
-                 jid :: jid:ljid(),
+-record(roster, {usj :: {jid:simple_bare_jid(), mod_roster:contact()},
+                 us :: jid:simple_bare_jid(),
+                 jid :: mod_roster:contact(),
                  name = <<>> :: binary(),
-                 subscription = none :: both | from | to | none | remove,
+                 subscription = none :: mod_roster:subscription_state(),
                  ask = none :: subscribe | unsubscribe | in | out | both | none,
-                 groups = [],
+                 groups = [] :: [binary()],
                  askmessage = <<>>,
-                 xs = []}).
+                 xs = [] :: [exml:element()]}).
 
 -record(roster_version, {us,
                         version}).

priv/migrations/mssql_6.2.0_6.2.1.sql

@@ -0,0 +1,10 @@
+DROP INDEX i_rosteru_server_user_jid ON rosterusers;
+DROP INDEX i_rosteru_server_user ON rosterusers;
+DROP INDEX i_rosteru_jid ON rosterusers;
+ALTER TABLE rosterusers
+ADD CONSTRAINT PK_rosterusers PRIMARY KEY CLUSTERED (server ASC, username ASC, jid ASC);
+
+DROP INDEX i_rosteru_jid ON rostergroups;
+ALTER TABLE rostergroups
+ALTER COLUMN grp VARCHAR(250),
+ADD CONSTRAINT PK_rostergroups PRIMARY KEY CLUSTERED (server ASC, username ASC, jid ASC, grp ASC);

priv/migrations/mysql_6.2.0_6.2.1.sql

@@ -0,0 +1,7 @@
+DROP INDEX i_rosteru_server_user_jid ON rosterusers;
+DROP INDEX i_rosteru_server_user ON rosterusers;
+DROP INDEX i_rosteru_jid ON rosterusers;
+ALTER TABLE rosterusers ADD PRIMARY KEY(server, username, jid);
+
+DROP INDEX i_rosterg_server_user_jid ON rostergroups;
+ALTER TABLE rostergroups MODIFY COLUMN grp VARCHAR(255), ADD PRIMARY KEY(server, username, jid, grp);

priv/migrations/pgsql_6.2.0_6.2.1.sql

@@ -0,0 +1,7 @@
+DROP INDEX i_rosteru_server_user_jid;
+DROP INDEX i_rosteru_server_user;
+DROP INDEX i_rosteru_jid;
+ALTER TABLE rosterusers ADD PRIMARY KEY (server, username, jid);
+
+DROP INDEX i_rosterg_server_user_jid;
+ALTER TABLE rostergroups ADD PRIMARY KEY (server, username, jid, grp);

priv/mssql2012.sql

@@ -278,12 +278,12 @@ GO
 SET ANSI_PADDING ON
 GO
 CREATE TABLE [dbo].[roster_version](
-        [server] [nvarchar](250) NOT NULL,
+    [server] [nvarchar](250) NOT NULL,
 	[username] [nvarchar](250) NOT NULL,
 	[version] [nvarchar](max) NOT NULL,
  CONSTRAINT [PK_roster_version_server_user] PRIMARY KEY CLUSTERED
 (
-        [server] ASC,
+    [server] ASC,
 	[username] ASC
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
@@ -302,14 +302,17 @@ CREATE TABLE [dbo].[rostergroups](
 	[server] [nvarchar](250) NOT NULL,
 	[username] [nvarchar](250) NOT NULL,
 	[jid] [nvarchar](250) NOT NULL,
-	[grp] [nvarchar](max) NOT NULL
-) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
-
+	[grp] [nvarchar](250) NOT NULL,
+ CONSTRAINT [PK_rostergroups] PRIMARY KEY CLUSTERED
+(
+    [server] ASC,
+	[username] ASC,
+	[jid] ASC,
+	[grp] ASC
+))
 GO
 SET ANSI_PADDING OFF
 GO
-CREATE INDEX i_rosterg_server_user_jid ON rostergroups (server, username, jid)
-GO
 /****** Object:  Table [dbo].[rosterusers]    Script Date: 9/17/2014 6:20:03 AM ******/
 SET ANSI_NULLS ON
 GO
@@ -326,21 +329,16 @@ CREATE TABLE [dbo].[rosterusers](
 	[ask] [char](1) NOT NULL,
 	[askmessage] [nvarchar](max) NOT NULL,
 	[created_at] [datetime] NOT NULL,
- CONSTRAINT [rosterusers$i_rosteru_server_user_jid] UNIQUE CLUSTERED
+ CONSTRAINT [PK_rosterusers] PRIMARY KEY CLUSTERED
 (
-	[server] ASC,
+    [server] ASC,
 	[username] ASC,
 	[jid] ASC
-)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
-) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
-
+))
 GO
+
 SET ANSI_PADDING OFF
 GO
-CREATE INDEX i_rosteru_server_user ON rosterusers (server, username)
-GO
-CREATE INDEX i_rosteru_jid ON rosterusers (jid)
-GO
 
 /****** Object:  Table [dbo].[users]    Script Date: 9/17/2014 6:20:03 AM ******/
 SET ANSI_NULLS ON

priv/mysql.sql

@@ -65,24 +65,27 @@ CREATE TABLE rosterusers (
     subscription character(1) NOT NULL,
     ask character(1) NOT NULL,
     askmessage text NOT NULL,
-    created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP
+    created_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (server, username, jid)
 ) CHARACTER SET utf8mb4
   ROW_FORMAT=DYNAMIC;
 
-CREATE UNIQUE INDEX i_rosteru_server_user_jid ON rosterusers (server, username, jid);
-CREATE INDEX i_rosteru_server_user ON rosterusers(server, username);
-CREATE INDEX i_rosteru_jid ON rosterusers(jid);
-
 CREATE TABLE rostergroups (
     server varchar(250) NOT NULL,
     username varchar(250) NOT NULL,
     jid varchar(250) NOT NULL,
-    grp text NOT NULL
+    grp varchar(250) NOT NULL,
+    PRIMARY KEY (server(150), username(200), jid, grp(150))
 ) CHARACTER SET utf8mb4
   ROW_FORMAT=DYNAMIC;
 
-CREATE INDEX i_rosterg_server_user_jid ON rostergroups(server, username, jid);
-
+CREATE TABLE roster_version (
+    server varchar(250),
+    username varchar(250),
+    version text NOT NULL,
+    PRIMARY KEY (server, username)
+) CHARACTER SET utf8mb4
+  ROW_FORMAT=DYNAMIC;
 
 CREATE TABLE vcard (
     username varchar(150),
@@ -180,14 +183,6 @@ CREATE TABLE private_storage (
 ) CHARACTER SET utf8mb4
   ROW_FORMAT=DYNAMIC;
 
-CREATE TABLE roster_version (
-    server varchar(250),
-    username varchar(250),
-    version text NOT NULL,
-    PRIMARY KEY (server, username)
-) CHARACTER SET utf8mb4
-  ROW_FORMAT=DYNAMIC;
-
 -- To update from 1.x:
 -- ALTER TABLE rosterusers ADD COLUMN askmessage text AFTER ask;
 -- UPDATE rosterusers SET askmessage = '';

priv/pg.sql

@@ -52,7 +52,6 @@ CREATE TABLE last (
 
 CREATE INDEX i_last_server_seconds ON last USING btree (server, seconds);
 
-
 CREATE TABLE rosterusers (
     server varchar(250) NOT NULL,
     username varchar(250) NOT NULL,
@@ -61,23 +60,24 @@ CREATE TABLE rosterusers (
     subscription character(1) NOT NULL,
     ask character(1) NOT NULL,
     askmessage text NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT now()
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    PRIMARY KEY (server, username, jid)
 );
 
-CREATE UNIQUE INDEX i_rosteru_server_user_jid ON rosterusers USING btree (server, username, jid);
-CREATE INDEX i_rosteru_server_user ON rosterusers USING btree (server, username);
-CREATE INDEX i_rosteru_jid ON rosterusers USING btree (jid);
-
-
 CREATE TABLE rostergroups (
     server varchar(250) NOT NULL,
     username varchar(250) NOT NULL,
     jid text NOT NULL,
-    grp text NOT NULL
+    grp text NOT NULL,
+    PRIMARY KEY (server, username, jid, grp)
 );
 
-CREATE INDEX i_rosterg_server_user_jid ON rostergroups USING btree (server, username, jid);
-
+CREATE TABLE roster_version (
+    server varchar(250),
+    username varchar(250),
+    version text NOT NULL,
+    PRIMARY KEY (server, username)
+);
 
 CREATE TABLE vcard (
     username varchar(150),
@@ -167,29 +167,6 @@ CREATE TABLE private_storage (
     PRIMARY KEY(server, username, namespace)
 );
 
-CREATE TABLE roster_version (
-    server varchar(250),
-    username varchar(250),
-    version text NOT NULL,
-    PRIMARY KEY (server, username)
-);
-
--- To update from 0.9.8:
--- CREATE SEQUENCE spool_seq_seq;
--- ALTER TABLE spool ADD COLUMN seq integer;
--- ALTER TABLE spool ALTER COLUMN seq SET DEFAULT nextval('spool_seq_seq');
--- UPDATE spool SET seq = DEFAULT;
--- ALTER TABLE spool ALTER COLUMN seq SET NOT NULL;
-
--- To update from 1.x:
--- ALTER TABLE rosterusers ADD COLUMN askmessage text;
--- UPDATE rosterusers SET askmessage = '';
--- ALTER TABLE rosterusers ALTER COLUMN askmessage SET NOT NULL;
-
--- To update from 2.0.0:
--- ALTER TABLE mam_message ADD COLUMN search_body text;
--- ALTER TABLE mam_muc_message ADD COLUMN search_body text;
-
 CREATE TYPE mam_behaviour AS ENUM('A', 'N', 'R');
 CREATE TYPE mam_direction AS ENUM('I','O');
 

rebar.config

@@ -53,9 +53,9 @@
  [
   %%% Stateless parsers
   {base16, "2.0.1"},
-  {exml, "3.4.0", {pkg, hexml}},
+  {exml, "3.4.1", {pkg, hexml}},
   {jiffy, "1.1.1"},
-  {jid, "2.0.0", {pkg, mongoose_jid}},
+  {jid, "2.1.0", {pkg, mongoose_jid}},
   {tomerl, "0.5.0"},
   {bbmustache, "1.12.2"},
   {erl_csv, "0.3.1"},