...
 
Commits (75)
......@@ -4,9 +4,10 @@ node_js:
- "0.8"
- "0.10"
# Create a travis enabled environment for the test suite to run in.
script: "make travisci"
# Create a travis memcached enabled environment for the test suite to run in and
# ensure that we test against localhost on travis-ci
services: memcache
env: MEMCACHED__HOST=localhost
# the `sevices: memcache` will start a memcached service on localhost
# and on the default port, but in order to test against multiple memcache
......
### 0.2.5
- Fix for two bugs in Issuelog #137 and #141
- Add new `failuresTimeout` option
### 0.2.4
- Tons of fixes have been made to the way we do error handling and failover,
this includes better reconnect, server failure detection, timeout handling
and much more.
- Introduction of a new `idle` timeout option.
- Documentation improvements.
### 0.2.3
- Added documentation for public api's
- new namespace option added that namespaces all your keys.
- minor parser fixes and some thrown errors.
### 0.2.2
- Support for touch command #86
- Fix for chunked responses from the server #84
### 0.2.1
- Supports for a queued callback limit so it would crash the process when we queue
to much callbacks. #81
### 0.2.0
- [breaking] We are now returning Error instances instead of strings for errors
- Dependency bump for a critical bug in our connection pool.
### 0.1.5
- Don't execute callbacks multiple times if the connection fails
- Parser fix for handling server responses that contain Memcached Protocol
keywords
- Make sure that the retry option is set correctly
### 0.1.4
- Added missing error listener to the 3rd-Eden/jackpot module, this prevents crashes
when it's unable to connect to a server.
### 0.1.3
- Handle Memcached responses that contain no value.
- Travis CI integration.
### 0.1.2
- Returning an error when the Memcached server issues a `NOT_STORED` response.
### 0.1.1
- Now using 3rd-Eden/jackpot as connection pool, this should give a more stable
connection.
### 0.1.0
- Storing numeric values are now returned as numeric values, they are no
longer strings.
### 0.0.12
- Added `.setEncoding` to the connections, this way UTF-8 chars will not be
chopped in to multiple pieces and breaking binary stored data or UTF-8 text
### 0.0.11
- Added more useful error messages instead of returning false. Please note
that they are still strings instead of Error instances (legacy)
### 0.0.10
- Compatibility with Node.js 0.8
- Don't saturate the Node process by retrying to connect if pool is full #43
- Minor code formatting
### 0.0.9
- Code style refactor, named the functions, removed tabs
- Added Mocha test suite
ALL_TESTS = $(shell find test -name '*.test.js')
test:
@./node_modules/.bin/mocha $(ALL_TESTS)
travisci:
MEMCACHED__HOST=localhost $(MAKE) test
doc:
dox --title "node-memcached" lib/* > doc/index.html
.PHONY: test doc
This diff is collapsed.
This diff is collapsed.
......@@ -12,9 +12,9 @@ var Ring = new HashRing(
);
// Return the server based on the key
process.stdout.write( Ring.getNode( "my-super-secret-cache-key" ) );
process.stdout.write( Ring.getNode( "hello-world" ) );
process.stdout.write( Ring.getNode( "my-super-secret-cache-key" ) );
process.stdout.write( Ring.get( "my-super-secret-cache-key" ) );
process.stdout.write( Ring.get( "hello-world" ) );
process.stdout.write( Ring.get( "my-super-secret-cache-key" ) );
// Different algorithms produce different hash maps. So choose wisely
var sha1Ring = new HashRing(
......@@ -30,6 +30,6 @@ var sha1Ring = new HashRing(
'sha1' // optional algorithm for key hashing
);
process.stdout.write( sha1Ring.getNode( "my-super-secret-cache-key" ) );
process.stdout.write( sha1Ring.getNode( "hello-world" ) );
process.stdout.write( sha1Ring.getNode( "my-super-secret-cache-key" ) );
process.stdout.write( sha1Ring.get( "my-super-secret-cache-key" ) );
process.stdout.write( sha1Ring.get( "hello-world" ) );
process.stdout.write( sha1Ring.get( "my-super-secret-cache-key" ) );
......@@ -2,7 +2,8 @@
var EventEmitter = require('events').EventEmitter
, spawn = require('child_process').spawn
, Utils = require('./utils');
, Utils = require('./utils')
, util = require('util');
exports.IssueLog = IssueLog; // connection issue handling
exports.Available = ping; // connection availablity
......@@ -37,7 +38,8 @@ function IssueLog (args) {
EventEmitter.call(this);
}
var issues = IssueLog.prototype = new EventEmitter;
util.inherits(IssueLog, EventEmitter);
var issues = IssueLog.prototype;
issues.log = function log (message) {
var issue = this;
......
......@@ -59,19 +59,22 @@ function Client (args, options) {
Utils.merge(this, options);
this.servers = servers;
this.HashRing = new HashRing(args, this.algorithm);
this.HashRing = new HashRing(args, this.algorithm, {
compatiblity: this.compatiblity
});
this.connections = {};
this.issues = [];
}
// Allows users to configure the memcached globally or per memcached client
Client.config = {
maxKeySize: 251 // max key size allowed by Memcached
maxKeySize: 250 // max key size allowed by Memcached
, maxExpiration: 2592000 // max expiration duration allowed by Memcached
, maxValue: 1048576 // max length of value allowed by Memcached
, activeQueries: 0
, maxQueueSize: -1
, algorithm: 'crc32' // hashing algorithm that is used for key mapping
, algorithm: 'md5' // hashing algorithm that is used for key mapping
, compatiblity: 'ketama' // hashring compatiblity
, poolSize: 10 // maximal parallel connections
, retries: 5 // Connection pool retries to pull connection from pool
......@@ -226,7 +229,7 @@ Client.config = {
keys.forEach(function fetchMultipleServers(key) {
var server = memcached.servers.length === 1
? memcached.servers[0]
: memcached.HashRing.getNode(key);
: memcached.HashRing.get(key);
if (map[server]){
map[server].push(key);
......@@ -284,7 +287,7 @@ Client.config = {
redundancy = this.HashRing.createRange(query.key, (this.redundancy + 1), true);
server = redundancy.shift();
} else {
server = this.HashRing.getNode(query.key);
server = this.HashRing.get(query.key);
}
}
}
......@@ -293,7 +296,7 @@ Client.config = {
// a server may not exist if the manager was never able to connect
// to any server.
if (!server || (server in this.issues && this.issues[server].failed)) {
return query.callback && memcached.makeCallback(query.callback,new Error('Server not available'));
return query.callback && memcached.makeCallback(query.callback,new Error(['Server at', server, 'not available'].join(' ')));
}
this.connect(server, function allocateMemcachedConnection(error, S) {
......@@ -303,18 +306,24 @@ Client.config = {
});
}
// check for issues
// S not set if unable to connect to server
if (!S) {
var S = {
serverAddress: server,
tokens: server.split(':').reverse()
}
var message = error || 'Unable to connect to server';
memcached.connectionIssue(message, S);
return query.callback && memcached.makeCallback(query.callback,new Error(message));
}
// Other errors besides inability to connect to server
if (error) {
memcached.connectionIssue(error.toString(), S);
return query.callback && memcached.makeCallback(query.callback,error);
}
if (!S) {
var message = 'Connect did not give a server';
memcached.connectionIssue(message);
return query.callback && memcached.makeCallback(query.callback,new Error(message));
}
if (S.readyState !== 'open') {
var message = 'Connection readyState is set to ' + S.readyState;
memcached.connectionIssue(message, S);
......@@ -369,6 +378,7 @@ Client.config = {
, failuresTimeout: this.failuresTimeout
, retry: this.retry
, remove: this.remove
, failOverServers: this.failOverServers || null
});
// proxy the events
......@@ -393,7 +403,7 @@ Client.config = {
if (this.failOverServers && this.failOverServers.length) {
memcached.HashRing.replaceServer(server, this.failOverServers.shift());
} else {
memcached.HashRing.removeServer(server);
memcached.HashRing.remove(server);
memcached.emit('failure', details);
}
}
......@@ -468,7 +478,7 @@ Client.config = {
}
, 'END': function end(tokens, dataSet, err, queue) {
if (!queue.length) queue.push(false);
if (!queue.length) queue.push(undefined);
return [FLUSH, true];
}
......@@ -638,6 +648,14 @@ Client.config = {
// only call transform the data once we are sure, 100% sure, that we valid
// response ending
if (S.responseBuffer.substr(S.responseBuffer.length - 2) === LINEBREAK) {
// Force v8 to re-allocate the responseBuffer and free the BufferStream
// chunks that were appended to it. This works around an issue in v8 where
// it doesn't free the appended strings which can cause poor GC behavior
// and make this function very slow for larger key values.
// See: https://code.google.com/p/v8/issues/detail?id=2869
S.responseBuffer = (' ' + S.responseBuffer).substr(1);
var chunks = S.responseBuffer.split(LINEBREAK);
if (this.debug) {
......@@ -836,7 +854,7 @@ Client.config = {
// add all responses to the array
(Array.isArray(results) ? results : [results]).forEach(function each(value) {
if (memcached.namespace.length) {
if (value && memcached.namespace.length) {
var ns_key = Object.keys(value)[0]
, newvalue = {};
......@@ -858,9 +876,11 @@ Client.config = {
memcached.command(function getMultiCommand(noreply) {
return {
callback: handle
, multi:true
, multi: true
, type: 'get'
, command: 'get ' + key.join(' ')
, key: keys
, validate: [['key', Array], ['callback', Function]]
};
}, server);
});
......@@ -995,9 +1015,10 @@ Client.config = {
// Small handler for incr and decr's
privates.incrdecr = function incrdecr(type, key, value, callback) {
var fullkey = this.namespace + key;
this.command(function incredecrCommand(noreply) {
return {
key: key
key: fullkey
, callback: callback
, value: value
, validate: [
......@@ -1007,7 +1028,7 @@ Client.config = {
]
, type: type
, redundancyEnabled: true
, command: [type, key, value].join(' ') +
, command: [type, fullkey, value].join(' ') +
(noreply ? NOREPLY : '')
};
});
......@@ -1019,9 +1040,10 @@ Client.config = {
// Deletes the keys from the servers
memcached.del = function del(key, callback){
var fullkey = this.namespace + key;
this.command(function deleteCommand(noreply) {
return {
key: key
key: fullkey
, callback: callback
, validate: [
['key', String]
......@@ -1029,7 +1051,7 @@ Client.config = {
]
, type: 'delete'
, redundancyEnabled: true
, command: 'delete ' + key +
, command: 'delete ' + fullkey +
(noreply ? NOREPLY : '')
};
});
......
......@@ -29,7 +29,17 @@ exports.validateArg = function validateArg (args, config) {
if (toString.call(value) !== '[object Array]') {
err = 'Argument "' + key + '" is not a valid Array.';
}
if (!err && key === 'key') {
for (var vKey in value) {
var vValue = value[vKey];
var result = validateKeySize(config, vKey, vValue);
if (result.err) {
err = result.err;
} else {
args.command = args.command.replace(vValue, result['value']);
}
}
}
break;
case Object:
......@@ -52,17 +62,11 @@ exports.validateArg = function validateArg (args, config) {
}
if (!err && key === 'key') {
if (value.length > config.maxKeySize) {
if (config.keyCompression){
args[key] = createHash('md5').update(value).digest('hex');
// also make sure you update the command
args.command = args.command.replace(value, args[key]);
} else {
err = 'Argument "' + key + '" is longer than the maximum allowed length of ' + config.maxKeySize;
}
} else if (/[\s\n\r]/.test(value)) {
err = 'The key should not contain any whitespace or new lines';
var result = validateKeySize(config, key, value);
if (result.err) {
err = result.err;
} else {
args.command = args.command.replace(value, result['value']);
}
}
break;
......@@ -75,16 +79,30 @@ exports.validateArg = function validateArg (args, config) {
});
if (err){
if(args.callback) args.callback(new Error(err));
if (args.callback) args.callback(new Error(err));
return false;
}
return true;
};
var validateKeySize = function validateKeySize(config, key, value) {
if (value.length > config.maxKeySize) {
if (config.keyCompression){
return { err: false, value: createHash('md5').update(value).digest('hex') };
} else {
return { err: 'Argument "' + key + '" is longer than the maximum allowed length of ' + config.maxKeySize };
}
} else if (/[\s\n\r]/.test(value)) {
return { err: 'The key should not contain any whitespace or new lines' };
} else {
return { err: false, value: value };
}
};
// a small util to use an object for eventEmitter
exports.fuse = function fuse (target, handlers) {
for(var i in handlers)
for (var i in handlers)
if (handlers.hasOwnProperty(i)){
target.on(i, handlers[i]);
}
......@@ -137,4 +155,4 @@ exports.escapeValue = function(value) {
//Unescapes escaped values by removing backslashes before line breaks
exports.unescapeValue = function(value) {
return value.replace(/\\(\r|\n)/g, '$1');
};
\ No newline at end of file
};
{
"name": "memcached"
, "version": "0.2.5"
, "author": "Arnout Kazemier"
, "description": "A fully featured Memcached API client, supporting both single and clustered Memcached servers through consistent hashing and failover/failure. Memcached is rewrite of nMemcached, which will be deprecated in the near future."
, "main": "index"
, "keywords":[
"memcached"
, "client"
, "hashing"
, "failover"
, "cluster"
, "nMemcached"
, "memcache"
, "cache"
, "nosql"
, "membase"
, "InnoDB memcached API"
]
, "directories": {
"lib": "./lib"
}
, "maintainers": [{
"name": "Arnout Kazemier"
, "email": "info@3rd-Eden.com"
, "url": "http://www.3rd-Eden.com"
}]
, "license": {
"type": "MIT"
, "url": "http://github.com/3rd-Eden/node-memcached/blob/master/LICENSE"
}
, "repository": {
"type": "git"
, "url" : "http://github.com/3rd-Eden/node-memcached.git"
}
, "dependencies": {
"hashring": "0.0.x"
, "jackpot": ">=0.0.6"
}
, "devDependencies": {
"mocha": "*"
, "should": "*"
}
, "scripts": {
"test": "./node_modules/.bin/mocha $(shell find test -name '*.test.js')"
}
"name": "memcached",
"version": "1.0.0",
"author": "Arnout Kazemier",
"description": "A fully featured Memcached API client, supporting both single and clustered Memcached servers through consistent hashing and failover/failure. Memcached is rewrite of nMemcached, which will be deprecated in the near future.",
"main": "index",
"keywords": [
"InnoDB memcached API",
"cache",
"client",
"cluster",
"failover",
"hashing",
"membase",
"memcache",
"memcached",
"nMemcached",
"nosql"
],
"license": "MIT",
"repository": {
"type": "git",
"url": "http://github.com/3rd-Eden/node-memcached.git"
},
"dependencies": {
"hashring": "1.0.x",
"jackpot": ">=0.0.6"
},
"devDependencies": {
"mocha": "*",
"should": "*",
"pre-commit": "*"
},
"scripts": {
"test": "mocha $(find test -name '*.test.js')"
}
}
......@@ -49,7 +49,7 @@ describe('Memcached connections', function () {
throw err;
});
};
assert.throws(noserver, /Server not available/);
assert.throws(noserver, new RegExp('Server at 127.0.1.1234 not available'));
memcached.end();
done();
});
......@@ -90,7 +90,7 @@ describe('Memcached connections', function () {
assert.deepEqual(memcached.issues[server].failed, true);
// Immediate request should not decrement failures
memcached.get('idontcare', function(err) {
assert.throws(function() { throw err }, /Server not available/);
assert.throws(function() { throw err }, /not available/);
assert.deepEqual(memcached.issues[server].failures, 5);
assert.deepEqual(memcached.issues[server].locked, true);
assert.deepEqual(memcached.issues[server].failed, true);
......@@ -129,7 +129,7 @@ describe('Memcached connections', function () {
assert.throws(function() { throw err }, /connect ECONNREFUSED/);
// Second request should not schedule another reconnect
memcached.get('idontcare', function (err) {
assert.throws(function() { throw err }, /Server not available/);
assert.throws(function() { throw err }, /not available/);
// Allow enough time to pass for a connection retries to occur
setTimeout(function() {
assert.deepEqual(reconnectAttempts, 1);
......@@ -161,7 +161,7 @@ describe('Memcached connections', function () {
assert.throws(function() { throw err }, /connect ECONNREFUSED/);
// Third request should find no servers
memcached.get('idontcare', function(err) {
assert.throws(function() { throw err }, /Server not available/);
assert.throws(function() { throw err }, /not available/);
// Give enough time for server to reconnect
setTimeout(function() {
// Server should be reconnected, but expect ECONNREFUSED
......@@ -224,7 +224,7 @@ describe('Memcached connections', function () {
var S = memcached.connections[common.servers.single].pool.pop();
S.emit('error', new Error('Dummy error'));
memcached.get('idontcare', function(err) {
assert.throws(function() { throw err; }, /Server not available/);
assert.throws(function() { throw err; }, /not available/);
done();
});
});
......
This diff is collapsed.
......@@ -43,7 +43,7 @@ describe('Memcached tests with Namespaces', function () {
assert.ok(!error);
ok.should.be.true;
answer.should.be.false;
assert.ok(answer===undefined);
// OK, now let's put it in with the namespace prepended
memcachedOther.set('test:' + testnr, message, 1000, function (error, ok) {
......@@ -104,4 +104,133 @@ describe('Memcached tests with Namespaces', function () {
});
});
});
/**
* In this case, these keys will be allocated to servers like below.
* test1,3,4 => :11211
* test5 => :11212
* test2 => :11213
*/
it('multi get from multi server with custom namespace (inc. cache miss)', function (done) {
var memcached = new Memcached(common.servers.multi, {
namespace: 'mySegmentedMemcached:'
})
, callbacks = 0;
// Load two namespaced variables into memcached
memcached.set('test1', 'test1answer', 1000, function (error, ok) {
++callbacks;
assert.ok(!error);
ok.should.be.true;
memcached.set('test2', 'test2answer', 1000, function (error, ok) {
++callbacks;
assert.ok(!error);
ok.should.be.true;
memcached.get(['test1', 'test2', 'test3', 'test4', 'test5'], function (error, answer) {
++callbacks;
assert.ok(typeof answer === 'object');
answer.test1.should.eql('test1answer');
answer.test2.should.eql('test2answer');
answer.should.not.have.key('test3');
answer.should.not.have.key('test4');
answer.should.not.have.key('test5');
memcached.end(); // close connections
assert.equal(callbacks, 3);
done();
});
});
});
});
it('should allow namespacing on delete', function(done) {
var memcached = new Memcached(common.servers.single, {
namespace:'someNamespace:'
}), callbacks = 0;
// put a value
memcached.set('test1', 'test1answer', 1000, function(error, ok) {
callbacks++;
assert.ok(!error);
ok.should.be.true;
// get it back
memcached.get('test1', function(error,answer) {
callbacks++;
assert.ok(typeof answer === 'string');
answer.should.eql('test1answer');
//delete it
memcached.del('test1', function(error) {
callbacks++;
assert.ok(!error);
// no longer there
memcached.get('test1', function(error,answer) {
callbacks++;
assert.ok(!error);
assert.ok(!answer);
memcached.end();
assert.equal(callbacks,4);
done();
});
});
});
});
});
it('should allow increment and decrement on namespaced values', function(done) {
var memcached = new Memcached(common.servers.single, {
namespace:'someNamespace:'
}), callbacks = 0;
// put a value
memcached.set('test1', 1, 1000, function(error, ok) {
callbacks++;
assert.ok(!error);
ok.should.be.true;
// increment it
memcached.incr('test1', 1, function(error) {
callbacks++;
assert.ok(!error);
// get it back
memcached.get('test1', function(error,answer) {
callbacks++;
assert.ok(!error);
assert.ok(typeof answer === 'number');
answer.should.be.eql(2);
// decrement it
memcached.decr('test1', 1, function(err) {
callbacks++;
assert.ok(!error);
// get it again
memcached.get('test1',function(error,answer) {
callbacks++;
assert.ok(!error);
assert.ok(typeof answer === 'number');
answer.should.be.eql(1);
//get rid of it
memcached.del('test1', function(error,answer) {
callbacks++;
assert.ok(!error);
memcached.end();
assert.equal(callbacks,6);
done();
});
});
});
});
});
});
});
});
......@@ -23,7 +23,7 @@ describe("Memcached TOUCH", function() {
, testnr = ++global.testnumbers
, callbacks = 0;
memcached.set("test:" + testnr, message, 1000, function(error, ok){
memcached.set("test:" + testnr, message, 1, function(error, ok){
++callbacks;
assert.ok(!error);
......@@ -40,7 +40,7 @@ describe("Memcached TOUCH", function() {
++callbacks;
assert.ok(!error);
answer.should.be.false;
assert.ok(answer===undefined);
memcached.end(); // close connections
assert.equal(callbacks, 3);
......