fix some shit

faulty query on 'like_pattern'
revisions ignored if pattern out of the original data

TODO:
probably have to reject data introduced via XLSX files (they contain a
weird json file)

Author: rrahir

src/util/spreadsheet/data_wrappers.py

@@ -22,8 +22,10 @@ def create_data_source_from_cmd(cmd):
     if cmd["type"] in ["CREATE_CHART", "UPDATE_CHART"]:
         return OdooChartCmdV16(cmd)
     elif cmd["type"] == "INSERT_PIVOT":
-        if version_gte("17.0"):
-            pass # TODO
+        if version_gte("saas~17.1"):
+            return InsertPivotCmdV171(cmd)
+        return InsertPivotCmdV16(cmd)
+    elif cmd["type"] == "RE_INSERT_PIVOT":
         return InsertPivotCmdV16(cmd)
     elif cmd["type"] == "INSERT_ODOO_LIST":
         return InsertListCmdV16(cmd)
@@ -489,3 +491,9 @@ def col_group_by(self):
     @col_group_by.setter
     def col_group_by(self, group_by):
         self.definition["metaData"]["colGroupBys"] = group_by
+
+
+class InsertPivotCmdV171(InsertPivotCmdV16):
+    @property
+    def id(self):
+        return self.cmd["pivotId"]

src/util/spreadsheet/fields.py

@@ -33,17 +33,14 @@ def remove_field_in_all_spreadsheets(cr, model, field):
 
 
 def remove_field(cr, model, field, data, revisions=()):
-    try:
-        spreadsheet = Spreadsheet(data)
-        _remove_field_from_filter_matching(cr, spreadsheet, model, field)
-        adapters = _remove_field_from_list(spreadsheet, model, field)
-        adapters += _remove_field_from_pivot(spreadsheet, model, field)
-        adapters += _remove_field_from_graph(spreadsheet, model, field)
-        adapters += _remove_field_from_view_link(spreadsheet, model, field)
-        spreadsheet.clean_empty_cells()
-        return spreadsheet.data, transform_revisions_data(revisions, *adapters)
-    except:
-        import ipdb;ipdb.set_trace()
+    spreadsheet = Spreadsheet(data)
+    _remove_field_from_filter_matching(cr, spreadsheet, model, field)
+    adapters = _remove_field_from_list(spreadsheet, model, field)
+    adapters += _remove_field_from_pivot(spreadsheet, model, field)
+    adapters += _remove_field_from_graph(spreadsheet, model, field)
+    adapters += _remove_field_from_view_link(spreadsheet, model, field)
+    spreadsheet.clean_empty_cells()
+    return spreadsheet.data, transform_revisions_data(revisions, *adapters)
 
 
 def _rename_function_fields(content, data_source_ids, functions, old, new):
@@ -335,25 +332,25 @@ def adapt_re_insert(cmd):
 
 
 def _remove_field_from_pivot(spreadsheet: Spreadsheet, model, field):
-    pivot_to_delete = [
+    pivots_to_delete = [
         pivot.id for pivot in spreadsheet.pivots if pivot.model == model and field in pivot_fields(pivot)
     ]
 
     adapters = remove_pivots(
         spreadsheet,
-        pivot_to_delete,
+        pivots_to_delete,
         lambda pivot: pivot.model == model and field in pivot_fields(pivot),
     )
     return adapters
 
 
 def _remove_field_from_graph(spreadsheet: Spreadsheet, model, field):
-    chart_to_delete = [
+    charts_to_delete = [
         chart.id for chart in spreadsheet.odoo_charts if chart.model == model and field in chart_fields(chart)
     ]
     return remove_odoo_charts(
         spreadsheet,
-        chart_to_delete,
+        charts_to_delete,
         lambda chart: chart.model == model and field in chart_fields(chart),
     )
 

src/util/spreadsheet/misc.py

@@ -45,10 +45,10 @@ def read_spreadsheet_initial_data(cr, like_pattern=""):
         cr.execute(
             """
             SELECT doc.id AS document_id, a.id AS attachment_id, a.db_datas
-            FROM documents_document doc
-        LEFT JOIN ir_attachment a ON a.id = doc.attachment_id
-            WHERE doc.handler='spreadsheet'
-            AND position(%s::bytea in db_datas) > 0
+              FROM documents_document doc
+         LEFT JOIN ir_attachment a ON a.id = doc.attachment_id
+             WHERE doc.handler='spreadsheet'
+               AND position(%s::bytea in db_datas) > 0
             """,
             [like_pattern],
         )
@@ -85,30 +85,41 @@ def _magic_spreadsheet_field(cr):
     return cr.fetchone()[0] and 'spreadsheet_binary_data' or 'data'
 
 def apply_in_all_spreadsheets(cr, like_pattern, callback):
-    # _logger.info("upgrading initial data and revisions")
     print("like pattern :   ", like_pattern)
     b = False
     # upgrade the initial data and all revisions based on it
     for attachment_id, res_model, res_id, db_datas in read_spreadsheet_initial_data(cr, like_pattern):
         print("attachment :   ", attachment_id)
         print("datas:   ", len(db_datas))
         b = True
-        revisions_data = []
-        revisions_ids = []
-        ## FIXME TODORAR batch the calls
-        for revision_id, commands in get_revisions(cr, res_model, res_id, like_pattern):
-            revisions_data.append(json.loads(commands))
-            revisions_ids.append(revision_id)
-        data, revisions = callback(db_datas, revisions_data)
+
+        data, _ = callback(db_datas, [])
         write_attachment(cr, attachment_id, data)
-        for revision_id, revision in zip(revisions_ids, revisions):
+
+        ## FIXME TODORAR batch the calls
+        ## FIXME  we have to pass in the revisions regardless of the base data stuff
+    # upgrade revisions
+    # regardless of res_model res_id
+    revisions_data = []
+    revisions_ids = []
+
+    # if like_pattern == "tax_audit":
+    #     import ipdb;ipdb.set_trace()
+    for revision_id, commands in get_revisions(cr, "res_model", "res_id", like_pattern):
+        revisions_data.append(json.loads(commands))
+        revisions_ids.append(revision_id)
+        data, revisions = callback({}, revisions_data)
+        # write_attachment(cr, attachment_id, data)
+        print("wooooooooooooooooot\n"*20)
+        print(revisions)
+        for rev_id, revision in zip(revisions_ids, revisions):
             cr.execute(
                 """
                 UPDATE spreadsheet_revision
                     SET commands=%s
                     WHERE id=%s
                 """,
-                [json.dumps(revision), revision_id],
+                [json.dumps(revision), rev_id],
             )
     if b:
         _logger.info("upgrading initial data and revisions")
@@ -132,8 +143,8 @@ def write_attachment(cr, attachment_id, data):
     cr.execute(
         """
         UPDATE ir_attachment
-            SET db_datas=%s
-            WHERE id=%s
+           SET db_datas=%s
+         WHERE id=%s
         """,
         [json.dumps(data).encode(), attachment_id]
     )
@@ -144,22 +155,19 @@ def get_revisions(cr, res_model, res_id, like_pattern):
         cr.execute(
             """
             SELECT id, commands
-            FROM spreadsheet_revision
-            WHERE commands LIKE %s
-            AND res_model=%s
-            AND res_id=%s
+              FROM spreadsheet_revision
+             WHERE commands LIKE %s
             """,
-            [like_pattern, res_model, res_id],
+            ['%' + like_pattern + '%'],
         )
     else:
         cr.execute(
             """
             SELECT id, commands
-            FROM spreadsheet_revision
-            WHERE commands LIKE %s
-            AND document_id=%s
+              FROM spreadsheet_revision
+             WHERE commands LIKE %s
             """,
-            [like_pattern, res_id],
+            [like_pattern],
         )
     return cr.fetchall()
 
@@ -330,20 +338,24 @@ def remove_odoo_charts(spreadsheet: Spreadsheet, chart_ids: List[str], insert_cm
     spreadsheet.delete_figures(*chart_ids)
 
     def adapt_create_chart(cmd):
+        # import ipdb;ipdb.set_trace()
+
         chart = create_data_source_from_cmd(cmd)
         if cmd["definition"]["type"].startswith("odoo_") and insert_cmd_predicate(chart):
             chart_ids.append(cmd["id"])
             return Drop
 
     def adapt_chart_cmd_with_id(cmd):
+        # import ipdb;ipdb.set_trace()
+
         if cmd["id"] in chart_ids:
             return Drop
 
     def adapt_global_filters(cmd):
         if cmd.get("chart"):
             for chart_id in chart_ids:
                 cmd["chart"].pop(chart_id, None)
-
+    # import ipdb;ipdb.set_trace()
     return (
         CommandAdapter("CREATE_CHART", adapt_create_chart),
         CommandAdapter("UPDATE_CHART", adapt_chart_cmd_with_id),

src/util/spreadsheet/models.py

@@ -125,7 +125,7 @@ def _remove_model_from_charts(model, spreadsheet: Spreadsheet):
 def _remove_model_from_filters(model, spreadsheet: Spreadsheet):
     global_filters = spreadsheet.global_filters
     to_delete = [
-        filter["id"] for filter in global_filters if filter["type"] == "relation" and filter["modelName"] == model
+        gFilter["id"] for gFilter in global_filters if gFilter["type"] == "relation" and gFilter["modelName"] == model
     ]
     spreadsheet.delete_global_filters(*to_delete)
 

src/util/spreadsheet/revisions.py

@@ -41,13 +41,14 @@ def transform_revisions(cr, revisions, *adapters: Iterable[CommandAdapter]):
             [json.dumps(data), revision_id],
         )
 def transform_revisions_data(revisions, *adapters: Iterable[CommandAdapter]):
+    # import ipdb;ipdb.set_trace()
     for data in revisions:
         commands = data.get("commands", [])
         if not commands:
             continue
         new_commands = transform_commands(commands, *adapters)
         if new_commands is None:
-            continue
+            new_commands = []
         # ensures it's jsonyfiable (converts tuples to lists) (probably should be removed)
         data["commands"] = json.loads(json.dumps(new_commands))
     return revisions