Improved parsing of diff [Edit script + SStubs]

Author: cedricrupb

code_diff/__init__.py

@@ -4,7 +4,7 @@
 from .ast     import parse_ast
 from .utils   import cached_property
 from .sstubs  import SStubPattern, classify_sstub
-from .gumtree import compute_edit_script
+from .gumtree import compute_edit_script, EditScript, Update
 
 
 # Main method --------------------------------------------------------
@@ -84,7 +84,11 @@ def statement_diff(self):
 
         return ASTDiff(self.config, source_stmt, target_stmt)
 
+    def root_diff(self):
+        return ASTDiff(self.config, ast_root(self.source_ast), ast_root(self.target_ast))
+
     def sstub_pattern(self):
+        
         if (parent_statement(self.config.statement_types, self.source_ast) is None
                 or parent_statement(self.config.statement_types, self.target_ast) is None):
             return SStubPattern.NO_STMT                
@@ -98,16 +102,18 @@ def edit_script(self):
 
         source_ast, target_ast = self.source_ast, self.target_ast
 
-        def is_statement_type(node_type):
-            return any(match_type(r, node_type) for r in self.config.statement_types)
+        if source_ast.type == target_ast.type and len(source_ast.children) == 0 and len(target_ast.children) == 0:
+            # Both nodes are tokens of the same type 
+            # Only an update is required
+            return EditScript([Update(source_ast, target_ast.text)])
+
+        # We need a common root to add to
+        while source_ast.type != target_ast.type: 
+            if source_ast.parent is None: break
+            if target_ast.parent is None: break
 
-        if not is_statement_type(source_ast.type) or not is_statement_type(target_ast.type):
-            # We need something where we can add to (root)
-            if source_ast.parent is not None:
-                source_ast = source_ast.parent
-            
-            if target_ast.parent is not None:
-                target_ast = target_ast.parent
+            source_ast = source_ast.parent
+            target_ast = target_ast.parent
 
         return compute_edit_script(source_ast, target_ast)
 
@@ -150,6 +156,15 @@ def is_statement_type(node_type):
     return parent_node
 
 
+def ast_root(ast):
+    parent_node = ast
+
+    while parent_node.parent is not None:
+        parent_node = parent_node.parent
+
+    return parent_node
+
+
 def tokenize_tree(ast):
     tokens = []
 
@@ -160,3 +175,18 @@ def tokenize_tree(ast):
         tokens.append(tokenize_tree(child))
 
     return " ".join(tokens)
+
+
+
+def is_compatible_root(root_candidate, source_ast):
+    return not equal_text(source_ast, root_candidate) and root_candidate.type != "block"
+
+
+def equal_text(source_ast, parent_ast):
+    source_position = source_ast.position
+    parent_position = parent_ast.position
+
+    if parent_position[0][0] < source_position[0][0]: return False
+    if source_position[1][0] < parent_position[1][0]: return False
+
+    return (source_position[0][1], source_position[1][1]) == (parent_position[0][1], parent_position[1][1])

code_diff/diff_utils.py

@@ -5,10 +5,11 @@
 
 class Hunk:
 
-    def __init__(self, lines, added_lines, rm_lines):
+    def __init__(self, lines, added_lines, rm_lines, header = None):
         self.lines       = lines
         self.added_lines = set(added_lines)
         self.rm_lines    = set(rm_lines)
+        self.header      = header
 
 
     @property
@@ -41,6 +42,10 @@ def before(self):
         return "".join(alines)
 
     def __repr__(self):
+
+        if self.header:
+            return self.header + "".join(self.lines)
+
         return "".join(self.lines)
 
 
@@ -55,7 +60,7 @@ def _parse_hunk(lines, start, end):
         if hline.startswith("+"): added_lines.append(i)
         if hline.startswith("-"): rm_lines.append(i)
 
-    return Hunk(hunk_lines, added_lines, rm_lines)
+    return Hunk(hunk_lines, added_lines, rm_lines, header = lines[start])
 
 
 hunk_pat = re.compile("@@ -(\d+)(,\d+)? \+(\d+)(,\d+)? @@.*")
@@ -136,5 +141,5 @@ def clean_hunk(hunk):
     added_lines = [l - start for l in hunk.added_lines if l >= start and l < end]
     rm_lines    = [l - start for l in hunk.rm_lines if l >= start and l < end]
 
-    return Hunk(new_lines, added_lines, rm_lines)
+    return Hunk(new_lines, added_lines, rm_lines, header = hunk.header)
 

code_diff/gumtree/__init__.py

@@ -2,7 +2,9 @@
 from .editmap  import gumtree_editmap
 from .chawathe import compute_chawathe_edit_script
 from .ops      import (Update, Insert, Delete, Move)
+from .ops      import EditScript
 from .ops      import serialize_script, deserialize_script
+from .ops      import json_serialize, json_deserialize
 
 # Edit script ----------------------------------------------------------------
 
@@ -27,15 +29,4 @@ def compute_edit_script(source_ast, target_ast, min_height = 1, max_size = 1000,
 # Update leaf ----------------------------------------------------------------
 
 def _update_leaf(source_ast, target_ast):
-    return Update(source_ast, target_ast.text)
-
-
-# Edit script ----------------------------------------------------------------
-
-class EditScript(list):
-
-    def __init__(self, operations):
-        super().__init__(operations)
-
-    def __repr__(self):
-        return serialize_script(self, indent = 2)
+    return Update(source_ast, target_ast.text)

code_diff/gumtree/isomap.py

@@ -1,6 +1,7 @@
 
 import heapq
 import itertools
+import math
 
 from collections import defaultdict
 
@@ -137,10 +138,25 @@ def _map_recursively(mapping, source_node, target_node):
 # Heuristic selection ----------------------------------------------------------------
 
 
+def source_distance(source_node, target_node):
+   
+    max_token_mover = 1000
+
+    line_mover_distance = source_node.position[0][0] - target_node.position[1][0]
+    line_mover_distance = line_mover_distance * max_token_mover
+
+    if line_mover_distance == 0:
+        token_mover_distance = min(abs(source_node.position[0][1] - target_node.position[0][1]), max_token_mover - 1) 
+        line_mover_distance += token_mover_distance
+
+    return -line_mover_distance
+
+
+
 def create_default_heuristic(isomorphic_mapping):
 
     def _heuristic(source_node, target_node):
-        return subtree_dice(source_node, target_node, isomorphic_mapping)
+        return (subtree_dice(source_node, target_node, isomorphic_mapping), source_distance(source_node, target_node))
 
     return _heuristic
 

code_diff/gumtree/ops.py

@@ -25,6 +25,16 @@ class Move(EditOperation):
 class Delete(EditOperation):
     pass
 
+# Edit script ----------------------------------------------------------------
+
+class EditScript(list):
+
+    def __init__(self, operations):
+        super().__init__(operations)
+
+    def __repr__(self):
+        return serialize_script(self, indent = 2)
+
 
 # Serialization --------------------------------
 
@@ -285,11 +295,11 @@ def json_serialize(edit_script):
             else: # Leaf node
                 new_node_str = ["%s:%s" % new_node, "T"]
 
-            edit_ops.append([operation_name, target_node_str, new_node_str])
+            edit_ops.append([operation_name, target_node_str, new_node_str, operation.position])
 
         elif operation_name == "Move":
 
-            new_node_str = _serialize_node(new_node_index, operation.node)
+            new_node_str = _json_serialize_node(new_node_index, operation.node)
 
             edit_ops.append([operation_name, target_node_str, new_node_str, operation.position])
 
@@ -299,4 +309,81 @@ def json_serialize(edit_script):
     return json.dumps(edit_ops)
 
 
-# Fast deserialize ----------------------------------------------------------------------
+# Fast deserialize ----------------------------------------------------------------------
+
+def _json_deserialize_node(node_index, node_info):
+
+    if not isinstance(node_info, list) and node_info != "T":
+        node_id = int(node_info[1:])
+        return node_index[node_id]
+
+    node_type, position = node_info[0], node_info[1:]
+    node_text = None
+
+    if ":" in node_type:
+        node_type, node_text = node_type.split(":", 1)
+
+    if len(position) == 4:
+        return DASTNode(node_type, ((position[0], position[1]), (position[2], position[3])), node_text)
+
+    return InsertNode(position[0], node_type, node_text)
+
+
+def _json_deserialize_node_constructor(node_index, cn_info):
+    node_type, node_id = cn_info
+    node_text = None
+
+    if ":" in node_type:
+        node_type, node_text = node_type.split(":", 1)
+
+    if node_id != "T":
+        node_id = int(node_id[1:])
+        node_index[node_id] = InsertNode(node_id, node_type, node_text)
+        return node_index[node_id]
+    
+    return InsertNode(node_id, node_type, node_text)
+
+
+def _json_deserialize_update(node_index, operation):
+    _, target, update = operation
+    target = _json_deserialize_node(node_index, target)
+    return Update(target, update)
+
+
+def _json_deserialize_insert(node_index, operation):
+    _, target, new_node, position = operation
+    target = _json_deserialize_node(node_index, target)
+    new_node = _json_deserialize_node_constructor(node_index, new_node)
+
+    return Insert(target, (new_node.type, new_node.text), position, new_node.node_id)
+
+
+def _json_deserialize_delete(node_index, operation):
+    return Delete(_json_deserialize_node(node_index, operation[1]))
+
+
+def _json_deserialize_move(node_index, operation):
+    _, target, move_node, position = operation
+    target = _json_deserialize_node(node_index, target)
+    move_node = _json_deserialize_node(node_index, move_node)
+    return Move(target, move_node, position)
+    
+
+DESERIALIZE = {
+    "Update" : _json_deserialize_update,
+    "Insert" : _json_deserialize_insert,
+    "Delete" : _json_deserialize_delete,
+    "Move"   : _json_deserialize_move
+}
+
+
+def json_deserialize(edit_json):
+    edit_ops = json.loads(edit_json)
+    output   = []
+    node_index = {}
+
+    for operation in edit_ops:
+        operation_name = operation[0]
+        output.append(DESERIALIZE[operation_name](node_index, operation))
+    
+    return EditScript(output)