10000 Initial commit of HyperLogLog commands · jrtkcoder/phpredis@110a993 · GitHub
[go: up one dir, main page]
Skip to content

Commit 110a993

Browse files
michael-grundermichael-grunder
committed
Initial commit of HyperLogLog commands
This is the initial commit of the HyperLogLog probabilistic counting command introduced in Redis. Support for the following commands is implemented * PFADD <key> <member1> <member2> ... <memberN> * PFCOUNT <key> * PFMERGE <dstkey> <srckey1> <srckey2> ... <srckeyN>
1 parent 02a67d5 commit 110a993

File tree

3 files changed

+292
-1
lines changed

3 files changed

+292
-1
lines changed

‎php_redis.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,11 @@ PHP_METHOD(Redis, hscan);
195195
PHP_METHOD(Redis, sscan);
196196
PHP_METHOD(Redis, zscan);
197197

198+
/* HyperLogLog commands */
199+
PHP_METHOD(Redis, pfadd);
200+
PHP_METHOD(Redis, pfcount);
201+
PHP_METHOD(Redis, pfmerge);
202+
198203
/* Reflection */
199204
PHP_METHOD(Redis, getHost);
200205
PHP_METHOD(Redis, getPort);

‎redis.c

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,11 @@ static zend_function_entry redis_functions[] = {
257257
PHP_ME(Redis, zscan, arginfo_kscan, ZEND_ACC_PUBLIC)
258258
PHP_ME(Redis, sscan, arginfo_kscan, ZEND_ACC_PUBLIC)
259259

260+
/* HyperLogLog commands */
261+
PHP_ME(Redis, pfadd, NULL, ZEND_ACC_PUBLIC)
262+
PHP_ME(Redis, pfcount, NULL, ZEND_ACC_PUBLIC)
263+
PHP_ME(Redis, pfmerge, NULL, ZEND_ACC_PUBLIC)
264+
260265
/* options */
261266
PHP_ME(Redis, getOption, NULL, ZEND_ACC_PUBLIC)
262267
PHP_ME(Redis, setOption, NULL, ZEND_ACC_PUBLIC)
@@ -7154,4 +7159,205 @@ PHP_METHOD(Redis, zscan) {
71547159
generic_scan_cmd(INTERNAL_FUNCTION_PARAM_PASSTHRU, TYPE_ZSCAN);
71557160
}
71567161

7162+
/*
7163+
* HyperLogLog based commands
7164+
*/
7165+
7166+
/* {{{ proto Redis::PFADD(string key, array elements) }}} */
7167+
PHP_METHOD(Redis, pfadd) {
7168+
zval *object;
7169+
RedisSock *redis_sock;
7170+
char *key;
7171+
int key_len, key_free, argc=1;
7172+
zval *z_mems, **z_mem;
7173+
HashTable *ht_mems;
7174+
HashPosition pos;
7175+
smart_str cmd = {0};
7176+
7177+
if(zend_parse_method_parameters(ZEND_NUM_ARGS() TSRMLS_CC, getThis(), "Osa",
7178+
&object, redis_ce, &key, &key_len, &z_mems)
7179+
==FAILURE)
7180+
{
7181+
RETURN_FALSE;
7182+
}
7183+
7184+
if(redis_sock_get(object, &redis_sock TSRMLS_CC, 0) < 0) {
7185+
RETURN_FALSE;
7186+
}
7187+
7188+
// Grab members as an array
7189+
ht_mems = Z_ARRVAL_P(z_mems);
7190+
7191+
// Total arguments we'll be sending
7192+
argc += zend_hash_num_elements(ht_mems);
7193+
7194+
// If the array was empty we can just exit
7195+
if(argc < 3) {
7196+
RETURN_FALSE;
7197+
}
7198+
7199+
// Start constructing our command
7200+
redis_cmd_init_sstr(&cmd, argc, "PFADD", sizeof("PFADD")-1);
7201+
7202+
// Prefix our key if we're prefixing
7203+
key_free = redis_key_prefix(redis_sock, &key, &key_len TSRMLS_CC);
7204+
redis_cmd_append_sstr(&cmd, key, key_len);
7205+
if(key_free) efree(key);
7206+
7207+
// Iterate over members we're adding
7208+
for(zend_hash_internal_pointer_reset_ex(ht_mems, &pos);
7209+
zend_hash_get_current_data_ex(ht_mems, (void**)&z_mem, &pos)==SUCCESS;
7210+
zend_hash_move_forward_ex(ht_mems, &pos))
7211+
{
7212+
char *mem;
7213+
int mem_len, val_free;
7214+
zval *z_tmp = NULL;
7215+
7216+
// Serialize if requested
7217+
val_free = redis_serialize(redis_sock, *z_mem, &mem, &mem_len TSRMLS_CC);
7218+
7219+
// Allow for non string members if we're not serializing
7220+
if(!val_free) {
7221+
if(Z_TYPE_PP(z_mem)==IS_STRING) {
7222+
mem = Z_STRVAL_PP(z_mem);
7223+
mem_len = Z_STRLEN_PP(z_mem);
7224+
} else {
7225+
MAKE_STD_ZVAL(z_tmp);
7226+
*z_tmp = **z_mem;
7227+
convert_to_string(z_tmp);
7228+
7229+
mem = Z_STRVAL_P(z_tmp);
7230+
mem_len = Z_STRLEN_P(z_tmp);
7231+
}
7232+
}
7233+
7234+
// Append this member
7235+
redis_cmd_append_sstr(&cmd, mem, mem_len);
7236+
7237+
// Free memory if we serialized or converted types
7238+
if(z_tmp) {
7239+
zval_dtor(z_tmp);
7240+
STR_FREE(z_tmp);
7241+
z_tmp = NULL;
7242+
} else if(val_free) {
7243+
efree(mem);
7244+
}
7245+
}
7246+
7247+
REDIS_PROCESS_REQUEST(redis_sock, cmd.c, cmd.len);
7248+
IF_ATOMIC() {
7249+
redis_1_response(INTERNAL_FUNCTION_PARAM_PASSTHRU, redis_sock, NULL, NULL);
7250+
}
7251+
REDIS_PROCESS_RESPONSE(redis_1_response);
7252+
}
7253+
7254+
/* {{{ proto Redis::PHFCOUNT(string key) }}}*/
7255+
PHP_METHOD(Redis, pfcount) {
7256+
zval *object;
7257+
RedisSock *redis_sock;
7258+
char *key, *cmd;
7259+
int key_len, cmd_len, key_free;
7260+
7261+
if(zend_parse_method_parameters(ZEND_NUM_ARGS() TSRMLS_CC, getThis(), "Os",
7262+
&object, redis_ce, &key, &key_len)==FAILURE)
7263+
{
7264+
RETURN_FALSE;
7265+
}
7266+
7267+
if(redis_sock_get(object, &redis_sock TSRMLS_CC, 0) < 0) {
7268+
RETURN_FALSE;
7269+
}
7270+
7271+
// Prefix key if neccisary and construct our command
7272+
key_free = redis_key_prefix(redis_sock, &key, &key_len TSRMLS_CC);
7273+
cmd_len = redis_cmd_format_static(&cmd, "PFCOUNT", "s", key, key_len);
7274+
if(key_free) efree(key);
7275+
7276+
REDIS_PROCESS_REQUEST(redis_sock, cmd, cmd_len);
7277+
IF_ATOMIC() {
7278+
redis_long_response(INTERNAL_FUNCTION_PARAM_PASSTHRU, redis_sock, 10000 NULL, NULL);
7279+
}
7280+
REDIS_PROCESS_RESPONSE(redis_long_response);
7281+
}
7282+
7283+
/* {{{ proto Redis::pfMerge(array keys) }}}*/
7284+
PHP_METHOD(Redis, pfmerge) {
7285+
zval *object;
7286+
RedisSock *redis_sock;
7287+
zval *z_keys, **z_key;
7288+
HashTable *ht_keys;
7289+
HashPosition pos;
7290+
smart_str cmd = {0};
7291+
int key_len, key_free, argc=1;
7292+
char *key;
7293+
7294+
if(zend_parse_method_parameters(ZEND_NUM_ARGS() TSRMLS_CC, getThis(), "Osa",
7295+
&object, redis_ce, &key, &key_len, &z_keys)==FAILURE)
7296+
{
7297+
RETURN_FALSE;
7298+
}
7299+
7300+
if(redis_sock_get(object, &redis_sock TSRMLS_CC, 0) < 0) {
7301+
RETURN_FALSE;
7302+
}
7303+
7304+
// Grab keys as an array
7305+
ht_keys = Z_ARRVAL_P(z_keys);
7306+
7307+
// Total arguments we'll be sending
7308+
argc += zend_hash_num_elements(ht_keys);
7309+
7310+
// If no keys were passed we can abort
7311+
if(argc<2) {
7312+
RETURN_FALSE;
7313+
}
7314+
7315+
// Initial construction of our command
7316+
redis_cmd_init_sstr(&cmd, argc, "PFMERGE", sizeof("PFMERGE")-1);
7317+
7318+
// Add our destination key (prefixed if necessary)
7319+
key_free = redis_key_prefix(redis_sock, &key, &key_len TSRMLS_CC);
7320+
redis_cmd_append_sstr(&cmd, key, key_len);
7321+
if(key_free) efree(key);
7322+
7323+
// Iterate our keys array
7324+
for(zend_hash_internal_pointer_reset_ex(ht_keys, &pos);
7325+
zend_hash_get_current_data_ex(ht_keys, (void**)&z_key, &pos)==SUCCESS;
7326+
zend_hash_move_forward_ex(ht_keys, &pos))
7327+
{
7328+
zval *z_tmp = NULL;
7329+
7330+
// Keys could look like a number
7331+
if(Z_TYPE_PP(z_key) == IS_STRING) {
7332+
key = Z_STRVAL_PP(z_key);
7333+
key_len = Z_STRLEN_PP(z_key);
7334+
} else {
7335+
MAKE_STD_ZVAL(z_tmp);
7336+
*z_tmp = **z_key;
7337+
convert_to_string(z_tmp);
7338+
7339+
key = Z_STRVAL_P(z_tmp);
7340+
key_len = Z_STRLEN_P(z_tmp);
7341+
}
7342+
7343+
// Prefix our key if necissary and append this key
7344+
key_free = redis_key_prefix(redis_sock, &key, &key_len TSRMLS_CC);
7345+
redis_cmd_append_sstr(&cmd, key, key_len);
7346+
if(key_free) efree(key);
7347+
7348+
// Free temporary zval if we converted
7349+
if(z_tmp) {
7350+
zval_dtor(z_tmp);
7351+
STR_FREE(z_tmp);
7352+
z_tmp = NULL;
7353+
}
7354+
}
7355+
7356+
REDIS_PROCESS_REQUEST(redis_sock, cmd.c, cmd.len);
7357+
IF_ATOMIC() {
7358+
redis_boolean_response(INTERNAL_FUNCTION_PARAM_PASSTHRU, redis_sock, NULL, NULL);
7359+
}
7360+
REDIS_PROCESS_RESPONSE(redis_boolean_response);
7361+
}
7362+
71577363
/* vim: set tabstop=4 softtabstops=4 noexpandtab shiftwidth=4: */

‎tests/TestRedis.php

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4492,7 +4492,7 @@ public function testSerialize() {
44924492
$vals = Array(1, 1.5, 'one', Array('here','is','an','array'));
44934493

44944494
// Test with no serialization at all
4495-
$this->assertTrue($this->redis->_serialize('test') === 'test');
4495+
$this->assertTrue($this->redis->_serialize('test') === 'test');
44964496
$this->assertTrue($this->redis->_serialize(1) === '1');
44974497
$this->assertTrue($this->redis->_serialize(Array()) === 'Array');
44984498
$this->assertTrue($this->redis->_serialize(new stdClass) === 'Object');
@@ -4806,6 +4806,86 @@ public function testZScan() {
48064806
$this->assertEquals(0, $i_p_score);
48074807
$this->assertEquals(0, $i_p_count);
48084808
}
4809+
4810+
//
4811+
// HyperLogLog (PF) commands
4812+
//
4813+
4814+
protected function createPFKey($str_key, $i_count) {
4815+
$arr_mems = Array();
4816+
for($i=0;$i<$i_count;$i++) {
4817+
$arr_mems[] = uniqid() . '-' . $i;
4818+
}
4819+
4820+
// Estimation by Redis
4821+
$this->redis->pfadd($str_key, $i_count);
4822+
}
4823+
4824+
public function testPFCommands() {
4825+
// Isn't available until 2.8.9
4826+
if(version_compare($this->version, "2.8.9", "lt")) {
4827+
$this->markTestSkipped();
4828+
return;
4829+
}
4830+
4831+
$str_uniq = uniqid();
4832+
$arr_mems = Array();
4833+
4834+
for($i=0;$i<1000;$i++) {
4835+
if($i%2 == 0) {
4836+
$arr_mems[] = $str_uniq . '-' . $i;
4837+
} else {
4838+
$arr_mems[] = $i;
4839+
}
4840+
}
4841+
4842+
// How many keys to create
4843+
$i_keys = 10;
4844+
4845+
// Iterate prefixing/serialization options
4846+
foreach(Array(Redis::SERIALIZER_NONE, Redis::SERIALIZER_PHP) as $str_ser) {
4847+
foreach(Array('', 'hl-key-prefix:') as $str_prefix) {
4848+
$arr_keys = Array();
4849+
4850+
// Now add for each key
4851+
for($i=0;$i<$i_keys;$i++) {
4852+
$str_key = "key:$i";
4853+
$arr_keys[] = $str_key;
4854+
4855+
// Clean up this key
4856+
$this->redis->del($str_key);
4857+
4858+
// Add to our cardinality set, and confirm we got a valid response
4859+
$this->assertTrue($this->redis->pfadd($str_key, $arr_mems));
4860+
4861+
// Grab estimated cardinality
4862+
$i_card = $this->redis->pfcount($str_key);
4863+
$this->assertTrue(is_int($i_card));
4864+
4865+
// Count should be close
4866+
$this->assertLess(abs($i_card-count($arr_mems)), count($arr_mems) * .1);
4867+
4868+
// The PFCOUNT on this key should be the same as the above returned response
4869+
$this->assertEquals($this->redis->pfcount($str_key), $i_card);
4870+
}
4871+
4872+
// Clean up merge key
4873+
$this->redis->del('pf-merge-key');
4874+
4875+
// Merge the counters
4876+
$this->assertTrue($this->redis->pfmerge('pf-merge-key', $arr_keys));
4877+
4878+
// Validate our merged count
4879+
$i_redis_card = $this->redis->pfcount('pf-merge-key');
4880+
4881+
// Merged cardinality should still be roughly 1000
4882+
$this->assertLess(abs($i_redis_card-count($arr_mems)), count($arr_mems) * .1);
4883+
4884+
// Clean up merge key
4885+
$this->redis->del('pf-merge-key');
4886+
}
4887+
}
4888+
}
48094889
}
48104890

48114891
exit(TestSuite::run("Redis_Test"));

0 commit comments

Comments
 (0)
0