switch to new hashing algorithm for hyperdb

README

+
+Uso di HyperDB con il piano R di A/I
+------------------------------------
+
+I database di noblogs sono distribuiti su piu' di un backend, utilizzando
+un sistema di consistent hashing che associa ogni blog_id ad un backend
+specifico.
+
+L'elenco dei backend e' specificato nel file /etc/noblogs/backends.
+
+Lo schema e' molto semplice e si basa sull'ID pubblico di ciascun server
+(es, latitanza:1, confino:3, etc). Ogni server ha un database 'noblogs_ID'
+(in /opt/noblogs/mysql), cui puo' accedere l'utente 'noblogsdb'.
+
+

db-backends.php

+<?php
+
+function noblogs_load_backends($db_config_file, &$hashptr) {
+  global $wpdb;
+
+  $backend_map = array();
+
+  $fp = @fopen($db_config_file, "r");
+  if (!$fp) {
+    die("Database backends not configured!");
+  }
+  while (($line = fgets($fp, 1024)) !== false) {
+    if ($line[0] == '#') {
+      continue;
+    }
+    $line_parts = explode(" ", rtrim($line));
+    $server_id = $line_parts[0];
+    $dataset = "backend_" . $server_id;
+    $backend_url = $line_parts[2];
+    $backend_url_data = parse_url($backend_url);
+    $backend = array(
+        "host" => $backend_url_data["host"] . ":" . $backend_url_data["port"],
+        "user" => $backend_url_data["user"],
+        "password" => $backend_url_data["pass"],
+        "name" => substr($backend_url_data["path"], 1),
+        "dataset" => $dataset,
+        "read" => 1, "write" => 1, "timeout" => 2
+        );
+    $wpdb->add_database($backend);
+    $hashptr->addTarget($dataset);
+    $backend_map[$dataset] = $backend;
+  }
+  fclose($fp);
+  return $backend_map;
+}

db-hash.php

+<?php
+
+require_once('flexihash.php');
+
+$wpdb_hash = new Flexihash();
+
+/* Hashing function to map blogs to databases.
+ *
+ * Implements a consistent hashing scheme using Flexihash.
+ */
+function noblogs_db_callback($query, $wpdb) {
+  global $wpdb_hash;
+  if (preg_match("/^{$wpdb->base_prefix}(\d+)_/", $wpdb->table, $matches)) {
+    $blog_id = $matches[1];
+    return $wpdb_hash->lookup($blog_id);
+  }
+}
+$wpdb->add_callback('noblogs_db_callback');
+  

flexihash.php

+<?php
+/**
+ * Flexihash - A simple consistent hashing implementation for PHP.
+ * 
+ * The MIT License
+ * 
+ * Copyright (c) 2008 Paul Annesley
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ * 
+ * @author Paul Annesley
+ * @link http://paul.annesley.cc/
+ * @copyright Paul Annesley, 2008
+ */
+
+/**
+ * A simple consistent hashing implementation with pluggable hash algorithms.
+ *
+ * @author Paul Annesley
+ * @package Flexihash
+ * @licence http://www.opensource.org/licenses/mit-license.php
+ */
+class Flexihash
+{
+
+	/**
+	 * The number of positions to hash each target to.
+	 *
+	 * @var int
+	 */
+	private $_replicas = 64;
+
+	/**
+	 * The hash algorithm, encapsulated in a Flexihash_Hasher implementation.
+	 * @var object Flexihash_Hasher
+	 */
+	private $_hasher;
+
+	/**
+	 * Internal counter for current number of targets.
+	 * @var int
+	 */
+	private $_targetCount = 0;
+
+	/**
+	 * Internal map of positions (hash outputs) to targets
+	 * @var array { position => target, ... }
+	 */
+	private $_positionToTarget = array();
+
+	/**
+	 * Internal map of targets to lists of positions that target is hashed to.
+	 * @var array { target => [ position, position, ... ], ... }
+	 */
+	private $_targetToPositions = array();
+
+	/**
+	 * Whether the internal map of positions to targets is already sorted.
+	 * @var boolean
+	 */
+	private $_positionToTargetSorted = false;
+
+	/**
+	 * Constructor
+	 * @param object $hasher Flexihash_Hasher
+	 * @param int $replicas Amount of positions to hash each target to.
+	 */
+	public function __construct(Flexihash_Hasher $hasher = null, $replicas = null)
+	{
+		$this->_hasher = $hasher ? $hasher : new Flexihash_Crc32Hasher();
+		if (!empty($replicas)) $this->_replicas = $replicas;
+	}
+
+	/**
+	 * Add a target.
+	 * @param string $target
+         * @param float $weight
+	 * @chainable
+	 */
+	public function addTarget($target, $weight=1)
+	{
+		if (isset($this->_targetToPositions[$target]))
+		{
+			throw new Flexihash_Exception("Target '$target' already exists.");
+		}
+
+		$this->_targetToPositions[$target] = array();
+
+		// hash the target into multiple positions
+		for ($i = 0; $i < round($this->_replicas*$weight); $i++)
+		{
+			$position = $this->_hasher->hash($target . $i);
+			$this->_positionToTarget[$position] = $target; // lookup
+			$this->_targetToPositions[$target] []= $position; // target removal
+		}
+
+		$this->_positionToTargetSorted = false;
+		$this->_targetCount++;
+
+		return $this;
+	}
+
+	/**
+	 * Add a list of targets.
+	 * @param array $targets
+         * @param float $weight
+	 * @chainable
+	 */
+	public function addTargets($targets, $weight=1)
+	{
+		foreach ($targets as $target)
+		{
+			$this->addTarget($target,$weight);
+		}
+
+		return $this;
+	}
+
+	/**
+	 * Remove a target.
+	 * @param string $target
+	 * @chainable
+	 */
+	public function removeTarget($target)
+	{
+		if (!isset($this->_targetToPositions[$target]))
+		{
+			throw new Flexihash_Exception("Target '$target' does not exist.");
+		}
+
+		foreach ($this->_targetToPositions[$target] as $position)
+		{
+			unset($this->_positionToTarget[$position]);
+		}
+
+		unset($this->_targetToPositions[$target]);
+
+		$this->_targetCount--;
+
+		return $this;
+	}
+
+	/**
+	 * A list of all potential targets
+	 * @return array
+	 */
+	public function getAllTargets()
+	{
+		return array_keys($this->_targetToPositions);
+	}
+
+	/**
+	 * Looks up the target for the given resource.
+	 * @param string $resource
+	 * @return string
+	 */
+	public function lookup($resource)
+	{
+		$targets = $this->lookupList($resource, 1);
+		if (empty($targets)) throw new Flexihash_Exception('No targets exist');
+		return $targets[0];
+	}
+
+	/**
+	 * Get a list of targets for the resource, in order of precedence.
+	 * Up to $requestedCount targets are returned, less if there are fewer in total.
+	 *
+	 * @param string $resource
+	 * @param int $requestedCount The length of the list to return
+	 * @return array List of targets
+	 */
+	public function lookupList($resource, $requestedCount)
+	{
+		if (!$requestedCount)
+			throw new Flexihash_Exception('Invalid count requested');
+
+		// handle no targets
+		if (empty($this->_positionToTarget))
+			return array();
+
+		// optimize single target
+		if ($this->_targetCount == 1)
+			return array_unique(array_values($this->_positionToTarget));
+
+		// hash resource to a position
+		$resourcePosition = $this->_hasher->hash($resource);
+
+		$results = array();
+		$collect = false;
+
+		$this->_sortPositionTargets();
+
+		// search values above the resourcePosition
+		foreach ($this->_positionToTarget as $key => $value)
+		{
+			// start collecting targets after passing resource position
+			if (!$collect && $key > $resourcePosition)
+			{
+				$collect = true;
+			}
+
+			// only collect the first instance of any target
+			if ($collect && !in_array($value, $results))
+			{
+				$results []= $value;
+			}
+
+			// return when enough results, or list exhausted
+			if (count($results) == $requestedCount || count($results) == $this->_targetCount)
+			{
+				return $results;
+			}
+		}
+
+		// loop to start - search values below the resourcePosition
+		foreach ($this->_positionToTarget as $key => $value)
+		{
+			if (!in_array($value, $results))
+			{
+				$results []= $value;
+			}
+
+			// return when enough results, or list exhausted
+			if (count($results) == $requestedCount || count($results) == $this->_targetCount)
+			{
+				return $results;
+			}
+		}
+
+		// return results after iterating through both "parts"
+		return $results;
+	}
+
+	public function __toString()
+	{
+		return sprintf(
+			'%s{targets:[%s]}',
+			get_class($this),
+			implode(',', $this->getAllTargets())
+		);
+	}
+
+	// ----------------------------------------
+	// private methods
+
+	/**
+	 * Sorts the internal mapping (positions to targets) by position
+	 */
+	private function _sortPositionTargets()
+	{
+		// sort by key (position) if not already
+		if (!$this->_positionToTargetSorted)
+		{
+			ksort($this->_positionToTarget, SORT_REGULAR);
+			$this->_positionToTargetSorted = true;
+		}
+	}
+
+}
+
+
+/**
+ * Hashes given values into a sortable fixed size address space.
+ *
+ * @author Paul Annesley
+ * @package Flexihash
+ * @licence http://www.opensource.org/licenses/mit-license.php
+ */
+interface Flexihash_Hasher
+{
+
+	/**
+	 * Hashes the given string into a 32bit address space.
+	 *
+	 * Note that the output may be more than 32bits of raw data, for example
+	 * hexidecimal characters representing a 32bit value.
+	 *
+	 * The data must have 0xFFFFFFFF possible values, and be sortable by
+	 * PHP sort functions using SORT_REGULAR.
+	 *
+	 * @param string
+	 * @return mixed A sortable format with 0xFFFFFFFF possible values
+	 */
+	public function hash($string);
+
+}
+
+
+/**
+ * Uses CRC32 to hash a value into a signed 32bit int address space.
+ * Under 32bit PHP this (safely) overflows into negatives ints.
+ *
+ * @author Paul Annesley
+ * @package Flexihash
+ * @licence http://www.opensource.org/licenses/mit-license.php
+ */
+class Flexihash_Crc32Hasher
+	implements Flexihash_Hasher
+{
+
+	/* (non-phpdoc)
+	 * @see Flexihash_Hasher::hash()
+	 */
+	public function hash($string)
+	{
+		return crc32($string);
+	}
+
+}
+
+
+/**
+ * Uses CRC32 to hash a value into a 32bit binary string data address space.
+ *
+ * @author Paul Annesley
+ * @package Flexihash
+ * @licence http://www.opensource.org/licenses/mit-license.php
+ */
+class Flexihash_Md5Hasher
+	implements Flexihash_Hasher
+{
+
+	/* (non-phpdoc)
+	 * @see Flexihash_Hasher::hash()
+	 */
+	public function hash($string)
+	{
+		return substr(md5($string), 0, 8); // 8 hexits = 32bit
+
+		// 4 bytes of binary md5 data could also be used, but
+		// performance seems to be the same.
+	}
+
+}
+
+
+/**
+ * An exception thrown by Flexihash.
+ *
+ * @author Paul Annesley
+ * @package Flexihash
+ * @licence http://www.opensource.org/licenses/mit-license.php
+ */
+class Flexihash_Exception extends Exception
+{
+}
+

tools/db-migrate-to-new-schema.php

+#!/opt/noblogs/cron/php-noblogs
+<?php
+
+// Load wordpress api.
+define('WP_CACHE',false);
+/** Setup WordPress environment */
+require_once('wp-load.php');
+require_once('db-config-new.php');
+
+
+function old_hash($dbid) {
+  if (($dbid % 2) == 0) {
+    return array('host' => '172.16.1.3',
+		 'port' => '3307',
+		 'user' => 'noblogs',
+		 'password' => 'n0bl0gst3st',
+		 'db' => 'noblogs_2');
+  } else {
+    return array('host' => '172.16.1.8',
+		 'port' => '3307',
+		 'user' => 'noblogsusr',
+		 'password' => 'n0bl0gsdb4xpw!',
+		 'db' => 'noblogs');
+  }
+}
+
+function new_hash($dbid) {
+  global $wpdb_hash;
+  global $wpdb_reverse_backend_map;
+  $lookup = $wpdb_hash->lookup($dbid);
+  $backend = $wpdb_reverse_backend_map[$lookup];
+  $result = array();
+  if (preg_match('/^(.*):([0-9]*)$/', $backend['host'], $matches)) {
+    $result['host'] = $matches[1];
+    $result['port'] = $matches[2];
+  }
+  $result['user'] = $backend['user'];
+  $result['password'] = $backend['password'];
+  $result['db'] = $backend['name'];
+  return $result;
+}
+
+function mysqlopts(&$attrs) {
+  return ("-h" . $attrs['host'] . " -P" . $attrs['port'] . " -u" . $attrs['user']
+	  . " '-p" . $attrs['password'] . "'");
+}
+
+function mysqlurl(&$attrs) {
+  return ("mysql://" . $attrs['user'] . "@" . $attrs['host']
+	  . ":" . $attrs['port'] . "/" . $attrs['db']);
+}
+
+function get_all_blogs() 
+{
+  global $wpdb;
+
+  $sql = "SELECT blog_id, domain, path FROM $wpdb->blogs WHERE public = 1 AND deleted = 0 AND archived = '0' ORDER BY domain, path";
+  $result = $wpdb->get_results($sql);
+  return ($result);
+}
+
+$new_counts = array();
+$moved_count = 0;
+$blogs = get_all_blogs();
+foreach ($blogs as $blog) {
+  $blog_id = $blog->blog_id;
+
+  $old_params = old_hash($blog_id);
+  $old_dburi = mysqlurl($old_params);
+  $new_params = new_hash($blog_id);
+  $new_dburi = mysqlurl($new_params);
+
+  if ($new_counts[$new_params['host']]) {
+    $new_counts[$new_params['host']] += 1;
+  } else { 
+    $new_counts[$new_params['host']] = 1;
+  }
+
+  if ($old_dburi != $new_dburi) {
+    echo "echo moving blog $blog_id from " . $old_params['host'] . " to " . $new_params['host'] . "\n";
+    echo "tables=\$(mysql " . mysqlopts($old_params) . " " . $old_params['db'] . " -NBe \"show tables like 'wp\\_" . $blog_id . "\\_%'\")\n";
+    echo "mysqldump --opt " . mysqlopts($old_params) . " " . $old_params['db'] . " \${tables} \\\n";
+    echo "  | mysql " . mysqlopts($new_params) . " " . $new_params['db'] . "\n";
+    $moved_count += 1;
+  } else {
+    echo "echo blog $blog_id stays on " . $old_params['host'] . "\n";
+  }
+
+}
+
+echo "\n\n\nBlog distribution:\n";
+print_r($new_counts);
+print "\n $moved_count blogs moved\n";
+