Commit ddbe78b5 authored by Eric Eastwood's avatar Eric Eastwood

Add message soft-delete (store message in another collection on delete)

The goal is to save the message text somewhere before deleting, so we can restore messages from false-reports, etc

This was originally a GitHub PR on the private project, https://github.com/troupe/gitter-webapp/pull/2575
parent c8d51677
# 19.16.0 - *upcoming*
- ...
Developer facing:
- Add message soft-delete (store message in another collection on delete), https://gitlab.com/gitlab-org/gitter/webapp/merge_requests/1225
# 19.15.0 - 2018-8-8
- Add feature toggle for embeds and disable by default, https://gitlab.com/gitlab-org/gitter/webapp/merge_requests/1223
......
......@@ -2,29 +2,32 @@
"use strict";
var env = require('gitter-web-env');
var stats = env.stats;
var errorReporter = env.errorReporter;
var logger = env.logger.get('chat');
var ChatMessage = require('gitter-web-persistence').ChatMessage;
var collections = require('gitter-web-utils/lib/collections');
var userService = require("gitter-web-users");
var processText = require('gitter-web-text-processor');
var Promise = require('bluebird');
var StatusError = require('statuserror');
var _ = require('lodash');
var mongooseUtils = require('gitter-web-persistence-utils/lib/mongoose-utils');
var groupResolver = require('./group-resolver');
var chatSearchService = require('./chat-search-service');
var unreadItemService = require('gitter-web-unread-items');
var markdownMajorVersion = require('gitter-markdown-processor').version.split('.')[0];
var getOrgNameFromTroupeName = require('gitter-web-shared/get-org-name-from-troupe-name');
var recentRoomService = require("gitter-web-rooms/lib/recent-room-service");
var mongoUtils = require('gitter-web-persistence-utils/lib/mongo-utils');
var securityDescriptorUtils = require('gitter-web-permissions/lib/security-descriptor-utils');
var mongoReadPrefs = require('gitter-web-persistence-utils/lib/mongo-read-prefs')
var chatSpamDetection = require('gitter-web-spam-detection/lib/chat-spam-detection');
const Promise = require('bluebird');
const _ = require('lodash');
const StatusError = require('statuserror');
const env = require('gitter-web-env');
const stats = env.stats;
const errorReporter = env.errorReporter;
const logger = env.logger.get('chat');
const mongooseUtils = require('gitter-web-persistence-utils/lib/mongoose-utils');
const mongoReadPrefs = require('gitter-web-persistence-utils/lib/mongo-read-prefs');
const persistence = require('gitter-web-persistence');
const ChatMessage = persistence.ChatMessage;
const ChatMessageBackup = persistence.ChatMessageBackup;
const mongoUtils = require('gitter-web-persistence-utils/lib/mongo-utils');
const securityDescriptorUtils = require('gitter-web-permissions/lib/security-descriptor-utils');
const chatSpamDetection = require('gitter-web-spam-detection/lib/chat-spam-detection');
const collections = require('gitter-web-utils/lib/collections');
const processText = require('gitter-web-text-processor');
const getOrgNameFromTroupeName = require('gitter-web-shared/get-org-name-from-troupe-name');
const groupResolver = require('./group-resolver');
const userService = require('gitter-web-users');
const chatSearchService = require('./chat-search-service');
const unreadItemService = require('gitter-web-unread-items');
const recentRoomService = require('gitter-web-rooms/lib/recent-room-service');
const markdownMajorVersion = require('gitter-markdown-processor').version.split('.')[0];
var useHints = true;
......@@ -127,7 +130,7 @@ function resolveMentions(troupe, user, parsedMessage) {
* NB: it is the callers responsibility to ensure that the user has permission
* to chat in the room
*/
exports.newChatMessageToTroupe = function(troupe, user, data) {
function newChatMessageToTroupe(troupe, user, data) {
// Keep this up here, set sent time asap to ensure order
var sentAt = new Date();
......@@ -167,8 +170,8 @@ exports.newChatMessageToTroupe = function(troupe, user, data) {
fromUserId: user.id,
toTroupeId: troupe.id,
sent: sentAt,
text: data.text, // Keep the raw message.
status: data.status, // Checks if it is a status update
text: data.text, // Keep the raw message.
status: data.status, // Checks if it is a status update
pub: isPublic || undefined, // Public room - useful for sampling
html: parsedMessage.html,
lang: parsedMessage.lang,
......@@ -218,10 +221,10 @@ exports.newChatMessageToTroupe = function(troupe, user, data) {
return chatMessage;
});
});
};
}
// Returns some recent public chats
exports.getRecentPublicChats = function() {
function getRecentPublicChats() {
var minRecentTime = Date.now() - RECENT_WINDOW_MILLISECONDS;
var minId = mongoUtils.createIdForTimestamp(minRecentTime);
......@@ -243,12 +246,12 @@ exports.getRecentPublicChats = function() {
return ChatMessage.aggregate(aggregation)
.read(mongoReadPrefs.secondaryPreferred)
.exec();
};
}
/**
* NB: It is the callers responsibility to ensure that the user has access to the room!
*/
exports.updateChatMessage = function(troupe, chatMessage, user, newText, callback) {
function updateChatMessage(troupe, chatMessage, user, newText, callback) {
return Promise.try(function() {
newText = newText || '';
......@@ -303,25 +306,25 @@ exports.updateChatMessage = function(troupe, chatMessage, user, newText, callbac
.thenReturn(chatMessage);
})
.nodeify(callback);
};
}
exports.findById = function(id, callback) {
function findById(id, callback) {
return ChatMessage.findById(id)
.exec()
.nodeify(callback);
};
}
exports.findByIdLean = function(id, fields) {
function findByIdLean(id, fields) {
return ChatMessage.findById(id, fields)
.lean()
.exec();
};
}
exports.findByIdInRoom = function(troupeId, id, callback) {
function findByIdInRoom(troupeId, id, callback) {
return ChatMessage.findOne({ _id: id, toTroupeId: troupeId })
.exec()
.nodeify(callback);
};
}
/**
* Returns a promise of chats with given ids
......@@ -329,7 +332,6 @@ exports.findByIdInRoom = function(troupeId, id, callback) {
function findByIds(ids, callback) {
return mongooseUtils.findByIds(ChatMessage, ids, callback);
}
exports.findByIds = findByIds;
/* This is much more cacheable than searching less than a date */
function getDateOfFirstMessageInRoom(troupeId) {
......@@ -345,7 +347,6 @@ function getDateOfFirstMessageInRoom(troupeId) {
return r[0].sent;
});
}
exports.getDateOfFirstMessageInRoom = getDateOfFirstMessageInRoom;
function findFirstUnreadMessageId(troupeId, userId) {
return unreadItemService.getFirstUnreadItem(userId, troupeId);
......@@ -368,7 +369,7 @@ function sentAfter(objectId) {
/**
* Returns a promise of messages
*/
exports.findChatMessagesForTroupe = function(troupeId, options, callback) {
function findChatMessagesForTroupe(troupeId, options = {}, callback) {
var limit = Math.min(options.limit || 50, 100);
var skip = options.skip || 0;
......@@ -492,9 +493,9 @@ exports.findChatMessagesForTroupe = function(troupeId, options, callback) {
.nodeify(callback);
};
}
exports.findChatMessagesForTroupeForDateRange = function(troupeId, startDate, endDate) {
function findChatMessagesForTroupeForDateRange(troupeId, startDate, endDate) {
var q = ChatMessage
.where('toTroupeId', troupeId)
.where('sent').gte(startDate)
......@@ -502,14 +503,14 @@ exports.findChatMessagesForTroupeForDateRange = function(troupeId, startDate, en
.sort({ sent: 'asc' });
return q.exec();
};
}
/**
* Search for messages in a room using a full-text index.
*
* Returns promise messages
*/
exports.searchChatMessagesForRoom = function(troupeId, textQuery, options) {
function searchChatMessagesForRoom(troupeId, textQuery, options) {
return chatSearchService.searchChatMessagesForRoom(troupeId, textQuery, options)
.then(function(searchResults) {
// We need to maintain the order of the original results
......@@ -539,31 +540,50 @@ exports.searchChatMessagesForRoom = function(troupeId, textQuery, options) {
return chatsOrdered;
});
});
}
};
function deleteMessage(message) {
// `_.omit` because of `Cannot update '__v' and '__v' at the same time` error
return mongooseUtils.upsert(ChatMessageBackup, { _id: message._id }, _.omit(message.toObject(), '__v'))
.then(() => {
message.remove();
});
}
exports.removeAllMessagesForUserIdInRoomId = function(userId, roomId) {
function removeAllMessagesForUserIdInRoomId(userId, roomId) {
return ChatMessage.find({ toTroupeId: roomId, fromUserId: userId })
.exec()
.then(function(messages) {
return Promise.map(messages, function(message) {
return message.remove();
}, { concurrency: 1 });
return Promise.map(messages, (message) => deleteMessage(message), { concurrency: 1 });
});
};
}
function deleteMessageFromRoom(troupe, chatMessage) {
return unreadItemService.removeItem(chatMessage.fromUserId, troupe, chatMessage)
.then(function() {
return chatMessage.remove();
})
.then(() => deleteMessage(chatMessage))
.return(null);
}
exports.deleteMessageFromRoom = deleteMessageFromRoom;
exports.testOnly = {
const testOnly = {
setUseHints: function(value) {
useHints = value;
}
};
module.exports = {
newChatMessageToTroupe,
getRecentPublicChats,
updateChatMessage,
findById,
findByIdLean,
findByIdInRoom,
findByIds,
getDateOfFirstMessageInRoom,
findChatMessagesForTroupe,
findChatMessagesForTroupeForDateRange,
searchChatMessagesForRoom,
removeAllMessagesForUserIdInRoomId,
deleteMessageFromRoom,
testOnly
};
......@@ -2,10 +2,13 @@
/*global describe:true, it: true, before:true */
"use strict";
var chatService = require('../lib/chat-service');
var fixtureLoader = require('gitter-web-test-utils/lib/test-fixtures');
var assert = require('assert');
var Promise = require('bluebird');
var fixtureLoader = require('gitter-web-test-utils/lib/test-fixtures');
var persistence = require('gitter-web-persistence');
var ChatMessage = persistence.ChatMessage;
var ChatMessageBackup = persistence.ChatMessageBackup;
var chatService = require('../lib/chat-service');
describe('chatService', function() {
......@@ -16,19 +19,14 @@ describe('chatService', function() {
var fixture = fixtureLoader.setup({
user1: {},
troupe1: {users: ['user1']},
message1: {
user: 'user1',
troupe: 'troupe1',
text: 'old_message',
sent: new Date("01/01/2014"),
pub: true
},
message2: {
user: 'user1',
troupe: 'troupe1',
text: 'new_message',
sent: new Date()
}
});
// Cleanup after every test so there isn't any contamination
afterEach(() => {
return Promise.all([
ChatMessage.remove({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id }),
ChatMessageBackup.remove({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id })
]);
});
describe('updateChatMessage', function() {
......@@ -81,7 +79,7 @@ describe('chatService', function() {
describe('Finding messages #slow', function() {
var chat1, chat2, chat3;
before(function() {
beforeEach(function() {
return chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'A' })
.then(function(chat) {
chat1 = chat.id;
......@@ -100,9 +98,9 @@ describe('chatService', function() {
return chatService.findChatMessagesForTroupe(fixture.troupe1.id, { aroundId: chat2 })
.then(function(chats) {
assert(chats.length >= 3);
assert.strictEqual(chats.filter(function(f) { return f.id == chat1; }).length, 1);
assert.strictEqual(chats.filter(function(f) { return f.id == chat2; }).length, 1);
assert.strictEqual(chats.filter(function(f) { return f.id == chat3; }).length, 1);
assert.strictEqual(chats.filter(function(f) { return f.id === chat1; }).length, 1);
assert.strictEqual(chats.filter(function(f) { return f.id === chat2; }).length, 1);
assert.strictEqual(chats.filter(function(f) { return f.id === chat3; }).length, 1);
});
});
......@@ -111,15 +109,15 @@ describe('chatService', function() {
chatService.findChatMessagesForTroupe(fixture.troupe1.id, { skip: 1, readPreference: 'primaryPreferred' }),
chatService.findChatMessagesForTroupe(fixture.troupe1.id, { }),
function(withSkip, withoutSkip) {
assert(withSkip.length > 2);
assert(withoutSkip.length > 2);
assert.strictEqual(withSkip.length, 2);
assert.strictEqual(withoutSkip.length, 3);
var lastItemWithoutSkip = withoutSkip[withoutSkip.length - 1];
var secondLastItemWithoutSkip = withoutSkip[withoutSkip.length - 2];
var lastItemWithSkip = withSkip[withSkip.length - 1];
// Last item without skip does not exist in with skip...
assert.deepEqual(withSkip.filter(function(f) { return f.id == lastItemWithoutSkip.id; }), []);
assert.deepEqual(withSkip.filter(function(f) { return f.id === lastItemWithoutSkip.id; }), []);
assert.strictEqual(secondLastItemWithoutSkip.id, lastItemWithSkip.id);
});
......@@ -167,13 +165,66 @@ describe('chatService', function() {
});
it('getRecentPublicChats #slow', function() {
fixtureLoader.disableMongoTableScans();
describe('getRecentPublicChats #slow', function() {
beforeEach(() => {
fixtureLoader.disableMongoTableScans();
return Promise.all([
chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'm1', status: true }),
chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'm2', status: true })
]);
})
it('finds messages', () => {
return chatService.getRecentPublicChats()
.then(function(chats) {
assert(chats.length >= 1);
});
});
});
describe('removeAllMessagesForUserIdInRoomId', () => {
it('should delete all messages for user ', function () {
return Promise.all([
chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'happy goat', status: true }),
chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'sad goat', status: true })
])
.then((chatMessages) => {
assert.strictEqual(chatMessages.length, 2);
})
.then(() => {
return chatService.removeAllMessagesForUserIdInRoomId(fixture.user1._id, fixture.troupe1._id);
})
.then(() => {
return Promise.props({
messages: ChatMessage.find({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id }),
messageBackups: ChatMessageBackup.find({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id })
})
})
.then(({ messages, messageBackups }) => {
assert.strictEqual(messages.length, 0);
assert.strictEqual(messageBackups.length, 2);
});
});
});
return chatService.getRecentPublicChats()
.then(function(chats) {
assert(chats.length >= 1);
});
describe('deleteMessageFromRoom', () => {
it('should delete message', function () {
return chatService.newChatMessageToTroupe(fixture.troupe1, fixture.user1, { text: 'happy goat', status: true })
.then((message) => {
return chatService.deleteMessageFromRoom(fixture.troupe1, message);
})
.then(() => {
return Promise.props({
messages: ChatMessage.find({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id }),
messageBackups: ChatMessageBackup.find({ toTroupeId: fixture.troupe1._id, fromUserId: fixture.user1._id })
});
})
.then(({ messages, messageBackups }) => {
assert.strictEqual(messages.length, 0);
assert.strictEqual(messageBackups.length, 1);
});
});
});
});
......@@ -2,14 +2,24 @@
var _ = require('lodash');
var Promise = require('bluebird');
var mongoose = require('gitter-web-mongoose-bluebird');
var mongoUtils = require('./mongo-utils');
var uniqueIds = require('mongodb-unique-ids');
var mongoReadPrefs = require('./mongo-read-prefs')
var mongoReadPrefs = require('./mongo-read-prefs');
var Schema = mongoose.Schema;
function idsIn(ids) {
return uniqueIds(ids).filter(function(id) { return !!id; });
}
function cloneSchema(schema) {
var tree = _.extend({}, schema.tree);
delete tree.id;
delete tree._id;
return new Schema(tree);
}
function hashList(list) {
if(!list) return null;
......@@ -298,6 +308,7 @@ module.exports = {
findByIdsLean: findByIdsLean,
addIdToLean: addIdToLean,
addIdToLeanArray: addIdToLeanArray,
cloneSchema: cloneSchema,
getEstimatedCountForId: getEstimatedCountForId,
getEstimatedCountForIds: getEstimatedCountForIds,
makeLastModifiedUpdater: makeLastModifiedUpdater
......
......@@ -12,6 +12,7 @@
"dependencies": {
"bluebird": "^3.5.1",
"gitter-web-env": "file:../env",
"gitter-web-mongoose-bluebird": "file:../mongoose-bluebird",
"lodash": "^3.2.0",
"mongodb": "^2.2.35",
"mongodb-unique-ids": "^0.2.0",
......
......@@ -76,6 +76,7 @@ var schemas = {
TroupeUser: require('./schemas/troupe-user-schema'),
UserSettings: require('./schemas/user-settings-schema'),
ChatMessage: require('./schemas/chat-message-schema'),
ChatMessageBackup: require('./schemas/chat-message-backup-schema'),
Event: require('./schemas/event-schema'),
OAuthClient: require('./schemas/oauth-client-schema'),
OAuthCode: require('./schemas/oauth-code-schema'),
......
'use strict';
var mongooseUtils = require('gitter-web-persistence-utils/lib/mongoose-utils');
var ChatMessage = require('./chat-message-schema');
var ChatMessageBackupSchema = mongooseUtils.cloneSchema(ChatMessage.ChatMessageSchema);
module.exports = {
install: function(mongooseConnection) {
var model = mongooseConnection.model('ChatMessageBackup', ChatMessageBackupSchema);
return {
model: model,
schema: ChatMessageBackupSchema
};
}
};
......@@ -35,6 +35,7 @@ ChatMessageSchema.schemaTypeName = 'ChatMessageSchema';
installVersionIncMiddleware(ChatMessageSchema);
module.exports = {
ChatMessageSchema: ChatMessageSchema,
install: function(mongooseConnection) {
var model = mongooseConnection.model('ChatMessage', ChatMessageSchema);
......
......@@ -15,6 +15,7 @@
"debug": "^2.2.0",
"gitter-web-env": "file:../env",
"gitter-web-mongoose-bluebird": "file:../mongoose-bluebird",
"gitter-web-persistence-utils": "file:../persistence-utils",
"lodash": "^3.2.0",
"mongoose": "^4.6.8",
"mongoose-number": "^0.1.1",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment