Weighted Levenshtein + refactored Levenshtein

tdebatty · tdebatty · commit c61fa9d3e0e0 · 2015-04-07T20:12:56.000+02:00
diff --git a/README.md b/README.md
@@ -43,6 +43,33 @@ public class MyApp {
 }
 ```
 
+## Weighted Levenshtein
+An implementation of Levenshtein that allows to define different weights for different character substitutions.
+
+```java
+import info.debatty.java.stringsimilarity.*;
+
+public class MyApp {
+
+    public static void main(String[] args) {
+        WeightedLevenshtein wl = new WeightedLevenshtein(
+      
+                new CharacterSubstitutionInterface() {
+                    public double cost(char c1, char c2) {
+                  
+                        // t and r are next to each other,
+                        // let's assign a lower cost to substitution
+                        if (c1 == 't' && c2 == 'r') {
+                            return 0.5;
+                        }
+                        
+                        return 1.0;
+                    }
+        });
+        System.out.println(wl.distanceAbsolute("String1", "Srring2"));
+    }
+```
+
 ## Damerau-Levenshtein
 Similar to Levenshtein, Damerau-Levenshtein distance is the minimum number of operations needed to transform one string into the other, where an operation is defined as an insertion, deletion, or substitution of a single character, or a **transposition of two adjacent characters**.
 
diff --git a/src/main/java/info/debatty/java/stringsimilarity/CharacterSubstitutionInterface.java b/src/main/java/info/debatty/java/stringsimilarity/CharacterSubstitutionInterface.java
@@ -0,0 +1,38 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2015 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package info.debatty.java.stringsimilarity;
+
+/**
+ * Used to indicate the cost of character substitution.
+ * 
+ * Cost should always be in [0.0 .. 1.0]
+ * For example, in an OCR application, cost('o', 'a') could be 0.4
+ * In a checkspelling application, cost('u', 'i') could be 0.4 because these are
+ * next to each other on the keyboard...
+ * @author Thibault Debatty
+ */
+public interface CharacterSubstitutionInterface {
+    public double cost(char c1, char c2);
+}
diff --git a/src/main/java/info/debatty/java/stringsimilarity/Levenshtein.java b/src/main/java/info/debatty/java/stringsimilarity/Levenshtein.java
@@ -13,6 +13,7 @@ public static void main (String[] args) {
         Levenshtein l = new Levenshtein();
         
         System.out.println(l.distanceAbsolute("My string", "My $tring"));
+        System.out.println(l.distanceAbsolute("My string", "M string2"));
         System.out.println(l.distance("My string", "My $tring"));
         System.out.println(l.similarity("My string", "My $tring"));
     }
@@ -24,7 +25,7 @@ public static int Distance(String s1, String s2) {
 
     @Override
     public double distance(String s1, String s2) {
-        return ((double) distanceAbsolute(s1, s2)) / Math.max(s1.length(), s2.length());
+        return (double) distanceAbsolute(s1, s2) / Math.max(s1.length(), s2.length());
         
     }
     
@@ -35,7 +36,7 @@ public double similarity(String s1, String s2) {
     
     /**
      * The Levenshtein distance, or edit distance, between two words is the 
-     * minimum number of single-character edits (i.e. insertions, deletions or 
+     * minimum number of single-character edits (insertions, deletions or 
      * substitutions) required to change one word into the other. 
      * 
      * http://en.wikipedia.org/wiki/Levenshtein_distance
@@ -45,57 +46,69 @@ public double similarity(String s1, String s2) {
      * It is zero if and only if the strings are equal.
      * If the strings are the same size, the Hamming distance is an upper bound 
      * on the Levenshtein distance.
-     * The Levenshtein distance between two strings is no greater than the sum 
-     * of their Levenshtein distances from a third string (triangle inequality).
+     * The Levenshtein distance verifies the triangle inequality (the distance 
+     * between two strings is no greater than the sum Levenshtein distances from
+     * a third string).
+     * 
+     * Implementation uses dynamic programming (Wagner–Fischer algorithm), with
+     * only 2 rows of data. The space requirement is thus O(m) and the algorithm
+     * runs in O(mn).
      * 
-     * @param s0
      * @param s1
+     * @param s2
      * @return 
      */
-    public int distanceAbsolute(String s0, String s1) {
-        int len0 = s0.length() + 1;
-        int len1 = s1.length() + 1;
-
-        // the array of distances
-        int[] cost = new int[len0];
-        int[] newcost = new int[len0];
-
-        // initial cost of skipping prefix in String s0
-        for (int i = 0; i < len0; i++) {
-            cost[i] = i;
+    public int distanceAbsolute(String s1, String s2) {
+        if (s1.equals(s2)){
+            return 0;
+        }
+        
+        if (s1.length() == 0) {
+            return s2.length();
+        }
+        
+        if (s2.length() == 0) {
+            return s1.length();
         }
 
-	// dynamicaly computing the array of distances
-        // transformation cost for each letter in s1
-        for (int j = 1; j < len1; j++) {
-
-            // initial cost of skipping prefix in String s1
-            newcost[0] = j - 1;
-
-            // transformation cost for each letter in s0
-            for (int i = 1; i < len0; i++) {
-
-                // matching current letters in both strings
-                int match = (s0.charAt(i - 1) == s1.charAt(j - 1)) ? 0 : 1;
+        // create two work vectors of integer distances
+        int[] v0 = new int[s2.length() + 1];
+        int[] v1 = new int[s2.length() + 1];
+        int[] vtemp;
 
-                // computing cost for each transformation
-                int cost_replace = cost[i - 1] + match;
-                int cost_insert = cost[i] + 1;
-                int cost_delete = newcost[i - 1] + 1;
+        // initialize v0 (the previous row of distances)
+        // this row is A[0][i]: edit distance for an empty s
+        // the distance is just the number of characters to delete from t
+        for (int i = 0; i < v0.length; i++) {
+            v0[i] = i;
+        }
+        
+        for (int i = 0; i < s1.length(); i++) {
+            // calculate v1 (current row distances) from the previous row v0
+            // first element of v1 is A[i+1][0]
+            //   edit distance is delete (i+1) chars from s to match empty t
+            v1[0] = i + 1;
 
-                // keep minimum cost
-                newcost[i] = Math.min(
-                        Math.min(cost_insert, cost_delete),
-                        cost_replace);
+            // use formula to fill in the rest of the row
+            for (int j = 0; j < s2.length(); j++) {
+                int cost = (s1.charAt(i) == s2.charAt(j)) ? 0 : 1;
+                v1[j + 1] = Math.min(
+                        v1[j] + 1,              // Cost of insertion
+                        Math.min(
+                                v0[j + 1] + 1,  // Cost of remove
+                                v0[j] + cost)); // Cost of substitution
             }
-
-            // swap cost/newcost arrays
-            int[] swap = cost;
-            cost = newcost;
-            newcost = swap;
+            
+            // copy v1 (current row) to v0 (previous row) for next iteration
+            //System.arraycopy(v1, 0, v0, 0, v0.length);
+            
+            // Flip references to current and previous row
+            vtemp = v0;
+            v0 = v1;
+            v1 = vtemp;
+                
         }
 
-        // the distance is the cost for transforming all letters in both strings
-        return cost[len0 - 1];
+        return v0[s2.length()];
     }
 }
diff --git a/src/main/java/info/debatty/java/stringsimilarity/WeightedLevenshtein.java b/src/main/java/info/debatty/java/stringsimilarity/WeightedLevenshtein.java
@@ -0,0 +1,118 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2015 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package info.debatty.java.stringsimilarity;
+
+/**
+ * Implementation of Levenshtein that allows to define different weights for
+ * different character substitutions.
+ * 
+ * @author Thibault Debatty
+ */
+public class WeightedLevenshtein implements StringSimilarityInterface {
+
+    /**
+     * @param args the command line arguments
+     */
+    public static void main(String[] args) {
+        WeightedLevenshtein wl = new WeightedLevenshtein(
+                new CharacterSubstitutionInterface() {
+                    public double cost(char c1, char c2) {
+                        if (c1 == 't' && c2 == 'r') {
+                            return 0.5;
+                        }
+                        return 1.0;
+                    }
+        });
+        System.out.println(wl.distanceAbsolute("String1", "Srring2"));
+    }
+    
+    private final CharacterSubstitutionInterface charsub;
+    
+    public WeightedLevenshtein(CharacterSubstitutionInterface charsub) {
+        this.charsub = charsub;
+    }
+    
+    public double distanceAbsolute(String s1, String s2) {
+        if (s1.equals(s2)){
+            return 0;
+        }
+        
+        if (s1.length() == 0) {
+            return s2.length();
+        }
+        
+        if (s2.length() == 0) {
+            return s1.length();
+        }
+
+        // create two work vectors of integer distances
+        double[] v0 = new double[s2.length() + 1];
+        double[] v1 = new double[s2.length() + 1];
+        double[] vtemp;
+
+        // initialize v0 (the previous row of distances)
+        // this row is A[0][i]: edit distance for an empty s
+        // the distance is just the number of characters to delete from t
+        for (int i = 0; i < v0.length; i++) {
+            v0[i] = i;
+        }
+        
+        for (int i = 0; i < s1.length(); i++) {
+            // calculate v1 (current row distances) from the previous row v0
+            // first element of v1 is A[i+1][0]
+            //   edit distance is delete (i+1) chars from s to match empty t
+            v1[0] = i + 1;
+
+            // use formula to fill in the rest of the row
+            for (int j = 0; j < s2.length(); j++) {
+                double cost = (s1.charAt(i) == s2.charAt(j)) ? 0 : charsub.cost(s1.charAt(i), s2.charAt(j));
+                v1[j + 1] = Math.min(
+                        v1[j] + 1,              // Cost of insertion
+                        Math.min(
+                                v0[j + 1] + 1,  // Cost of remove
+                                v0[j] + cost)); // Cost of substitution
+            }
+            
+            // copy v1 (current row) to v0 (previous row) for next iteration
+            //System.arraycopy(v1, 0, v0, 0, v0.length);
+            
+            // Flip references to current and previous row
+            vtemp = v0;
+            v0 = v1;
+            v1 = vtemp;
+                
+        }
+
+        return v0[s2.length()];
+    }
+
+    public double similarity(String s1, String s2) {
+        return 1.0 - distance(s1, s2);
+    }
+
+    public double distance(String s1, String s2) {
+        return (double) distanceAbsolute(s1, s2) / Math.max(s1.length(), s2.length());
+    }
+}
