almost but concurrent access

Author: LucasLefevre

src/util/fields.py

@@ -37,7 +37,7 @@ def make_index_name(table_name, column_name):
         return "%s_%s_index" % (table_name, column_name)
 
 
-from .. import spreadsheet
+from . import spreadsheet
 from .const import ENVIRON
 from .context import adapt_context, clean_context
 from .domains import _adapt_one_domain, _replace_path, _valid_path_to, adapt_domains

src/util/models.py

@@ -9,7 +9,7 @@
 import logging
 import re
 
-from .. import spreadsheet
+from . import spreadsheet
 from .const import ENVIRON
 from .fields import IMD_FIELD_PATTERN, remove_field
 from .helpers import _ir_values_value, _validate_model, model_of_table, table_of_model

src/util/spreadsheet/fields.py

@@ -17,7 +17,7 @@
 from odoo.upgrade.util.domains import _adapt_one_domain
 
 def rename_field_in_all_spreadsheets(cr, model, old_value, new_value):
-    apply_in_all_spreadsheets(cr, lambda data, revisions_data: rename_field(cr, model, old_value, new_value, data, revisions_data))
+    apply_in_all_spreadsheets(cr, old_value, (lambda data, revisions_data: rename_field(cr, model, old_value, new_value, data, revisions_data)))
 
 def rename_field(cr, model, old, new, data, revisions = ()):
     spreadsheet = Spreadsheet(data)
@@ -29,7 +29,7 @@ def rename_field(cr, model, old, new, data, revisions = ()):
     return spreadsheet.data, transform_revisions_data(revisions, *adapters)
 
 def remove_field_in_all_spreadsheets(cr, model, field):
-    apply_in_all_spreadsheets(cr, lambda data, revisions_data: remove_field(cr, model, field, data, revisions_data))
+    apply_in_all_spreadsheets(cr, model, (lambda data, revisions_data: remove_field(cr, model, field, data, revisions_data)))
 
 def remove_field(cr, model, field, data, revisions=()):
     spreadsheet = Spreadsheet(data)

src/util/spreadsheet/misc.py

@@ -1,6 +1,7 @@
 
 import re
 import json
+import logging
 
 from typing import Union, Callable, Dict, List
 
@@ -14,6 +15,8 @@
 from .revisions import CommandAdapter, Drop
 from odoo.upgrade.util.misc import version_gte
 
+_logger = logging.getLogger(__name__)
+
 
 def read_spreadsheet_attachments(cr, like_pattern="%"):
     yield from read_spreadsheet_initial_data(cr, like_pattern)
@@ -25,15 +28,16 @@ def read_spreadsheet_snapshots(cr, like_pattern="%"):
         """
         SELECT id, res_model, res_id, db_datas
           FROM ir_attachment
-         WHERE res_model IN ['spreadsheet.dashboard', 'documents.document']
+         WHERE res_model IN ('spreadsheet.dashboard', 'documents.document')
            AND res_field = 'spreadsheet_snapshot'
            AND db_datas LIKE %s
         """,
         [like_pattern],
     )
+    # TODO LIKE probably doesn't work because the field is of type bytea
     for attachment_id, res_model, res_id, db_datas in cr.fetchall():
         if db_datas:
-            yield attachment_id, res_model, res_id, json.loads(db_datas)
+            yield attachment_id, res_model, res_id, json.loads(db_datas.tobytes())
 
 
 def read_spreadsheet_initial_data(cr, like_pattern="%"):
@@ -50,31 +54,33 @@ def read_spreadsheet_initial_data(cr, like_pattern="%"):
     # TODO there are excel files in there!
     for document_id, attachment_id, db_datas in cr.fetchall():
         if db_datas:
-            yield attachment_id, "documents.document", document_id, json.loads(db_datas)
+            # print(db_datas)
+            yield attachment_id, "documents.document", document_id, json.loads(db_datas.tobytes())
     data_field = "spreadsheet_binary_data" if version_gte("saas~16.3") else "data"
     cr.execute(
         """
         SELECT id, res_model, res_id, db_datas
           FROM ir_attachment
-         WHERE res_model IN = 'spreadsheet.dashboard'
+         WHERE res_model = 'spreadsheet.dashboard'
            AND res_field = %s
            AND db_datas LIKE %s
         """,
         [data_field, like_pattern],
     )
     for attachment_id, res_model, res_id, db_datas in cr.fetchall():
         if db_datas:
-            yield attachment_id, res_model, res_id, json.loads(db_datas)
+            yield attachment_id, res_model, res_id, json.loads(db_datas.tobytes())
 
 def apply_in_all_spreadsheets(cr, like_pattern, callback):
+    _logger.info("upgrading initial data and revisions")
     # upgrade the initial data and all revisions based on it
     for attachment_id, res_model, res_id, db_datas in read_spreadsheet_initial_data(cr):
         revisions_data = []
         revisions_ids = []
         for revision_id, commands in get_revisions(cr, res_model, res_id, like_pattern):
             revisions_data.append(json.loads(commands))
             revisions_ids.append(revision_id)
-        data, revisions = callback(cr, db_datas, revisions_data)
+        data, revisions = callback(db_datas, revisions_data)
         write_attachment(cr, attachment_id, data)
         for revision_id, revision in zip(revisions_ids, revisions):
             cr.execute(
@@ -85,22 +91,25 @@ def apply_in_all_spreadsheets(cr, like_pattern, callback):
                 """,
                 [json.dumps(revision), revision_id],
             )
+    _logger.info("upgrading snapshots")
     # upgrade snapshots
     for attachment_id, _res_model, _res_id, db_datas in read_spreadsheet_snapshots(cr):
         data, revisions = callback(cr, db_datas, [])
         write_attachment(cr, attachment_id, data)
 
 
 def write_attachment(cr, attachment_id, data):
+    _logger.info("replacing attachment %s", attachment_id)
     cr.execute(
         """
         UPDATE ir_attachment
             SET db_datas=%s
             WHERE id=%s
         """,
-        [json.dumps(data), attachment_id]
+        [json.dumps(data).encode(), attachment_id]
     )
 
+
 def get_revisions(cr, res_model, res_id, like_pattern):
     if version_gte("16.0"):
         cr.execute(
@@ -134,8 +143,9 @@ def upgrade_data(cr, upgrade_callback):
                 SET db_datas=%s
                 WHERE id=%s
             """,
-            [json.dumps(upgraded_data), attachment_id],
+            [json.dumps(upgraded_data).encode(), attachment_id],
         )
+    _logger.info("spreadsheet json data upgraded")
 
 
 def transform_data_source_functions(content, data_source_ids, functions, adapter):

src/util/spreadsheet/models.py

@@ -5,7 +5,7 @@
 
 
 def rename_model_in_all_spreadsheets(cr, old_value, new_value):
-    apply_in_all_spreadsheets(cr, lambda data, revisions_data: rename_model(old_value, new_value, data, revisions_data))
+    apply_in_all_spreadsheets(cr, old_value, (lambda data, revisions_data: rename_model(old_value, new_value, data, revisions_data)))
 
 # TODO remove cr argument
 def rename_model(old, new, data, revisions = ()):
@@ -18,7 +18,7 @@ def rename_model(old, new, data, revisions = ()):
     return spreadsheet.data, transform_revisions_data(revisions, *adapters)
 
 def remove_model_in_all_spreadsheets(cr, model):
-    apply_in_all_spreadsheets(cr, lambda data, revisions_data: remove_model(model, data, revisions_data))
+    apply_in_all_spreadsheets(cr, model, (lambda data, revisions_data: remove_model(model, data, revisions_data)))
 
 def remove_model(model: str, data, revisions = ()) -> str:
     spreadsheet = Spreadsheet(data)

src/util/spreadsheet/o_spreadsheet.py

@@ -44,16 +44,16 @@ def _create_empty_workbook(sheet_name: str = "Sheet1") -> dict:
     }
 
 
-def upgrade_data(data, upgrade_functions, to_version, version_field="version"):
-    data = load(data)
-    upgrade_functions.sort(key=lambda x: x[0])
-    for upgrade_version, upgrade in upgrade_functions:
-        if data.get(version_field, 0) < upgrade_version:
-            upgrade(data)
-            data[version_field] = upgrade_version
-        if upgrade_version == to_version:
-            return data
-    return data
+# def upgrade_data(data, upgrade_functions, to_version, version_field="version"):
+#     data = load(data)
+#     upgrade_functions.sort(key=lambda x: x[0])
+#     for upgrade_version, upgrade in upgrade_functions:
+#         if data.get(version_field, 0) < upgrade_version:
+#             upgrade(data)
+#             data[version_field] = upgrade_version
+#         if upgrade_version == to_version:
+#             return data
+#     return data
 
 
 # Reference of a column header (eg. A, AB)