Fail loudly when file is missing and change defaults to aggressively fail instead of passively

Author: vishalbollu

pkg/workloads/lib/storage/local.py

@@ -38,7 +38,7 @@ def _get_or_create_path(self, key):
         p.parent.mkdir(parents=True, exist_ok=True)
         return p
 
-    def _get_path_if_exists(self, key, allow_missing=True):
+    def _get_path_if_exists(self, key, allow_missing=False):
         p = Path(os.path.join(self.base_dir, key))
         if not p.exists() and allow_missing:
             return None
@@ -69,7 +69,7 @@ def put_json(self, obj, key):
         f = self._get_or_create_path(key)
         f.write_text(json.dumps(obj))
 
-    def get_json(self, key, allow_missing=True):
+    def get_json(self, key, allow_missing=False):
         f = self._get_path_if_exists(key, allow_missing)
         if f is None:
             return None
@@ -79,7 +79,7 @@ def put_msgpack(self, obj, key):
         f = self._get_or_create_path(key)
         f.write_bytes(msgpack.dumps(obj))
 
-    def get_msgpack(self, key, allow_missing=True):
+    def get_msgpack(self, key, allow_missing=False):
         f = self._get_path_if_exists(key, allow_missing)
         if f is None:
             return None
@@ -89,7 +89,7 @@ def put_pyobj(self, obj, key):
         f = self._get_or_create_path(key)
         f.write_bytes(pickle.dumps(obj))
 
-    def get_pyobj(self, key, bucket, allow_missing=True):
+    def get_pyobj(self, key, bucket, allow_missing=False):
         f = self._get_path_if_exists(key, allow_missing)
         if f is None:
             return None

pkg/workloads/lib/storage/s3.py

@@ -114,13 +114,13 @@ def _get_matching_s3_keys_generator(self, prefix="", suffix=""):
     def _upload_string_to_s3(self, string, key):
         self.s3.put_object(Bucket=self.bucket, Key=key, Body=string)
 
-    def _read_bytes_from_s3(self, key, allow_missing=True):
+    def _read_bytes_from_s3(self, key, allow_missing=False):
         try:
             byte_array = self.s3.get_object(Bucket=self.bucket, Key=key)["Body"].read()
         except self.s3.exceptions.NoSuchKey as e:
             if allow_missing:
                 return None
-            raise e
+            raise CortexException("bucket " + self.bucket, "key " + key) from e
 
         return byte_array.strip()
 
@@ -130,7 +130,7 @@ def search(self, prefix="", suffix=""):
     def put_json(self, obj, key):
         self._upload_string_to_s3(json.dumps(obj), key)
 
-    def get_json(self, key, allow_missing=True):
+    def get_json(self, key, allow_missing=False):
         obj = self._read_bytes_from_s3(key, allow_missing).decode("utf-8")
         if obj is None:
             return None
@@ -139,7 +139,7 @@ def get_json(self, key, allow_missing=True):
     def put_msgpack(self, obj, key):
         self._upload_string_to_s3(msgpack.dumps(obj), key)
 
-    def get_msgpack(self, key, allow_missing=True):
+    def get_msgpack(self, key, allow_missing=False):
         obj = self._read_bytes_from_s3(key, allow_missing)
         if obj == None:
             return None
@@ -148,7 +148,7 @@ def get_msgpack(self, key, allow_missing=True):
     def put_pyobj(self, obj, key):
         self._upload_string_to_s3(pickle.dumps(obj), key)
 
-    def get_pyobj(self, key, allow_missing=True):
+    def get_pyobj(self, key, allow_missing=False):
         obj = self._read_bytes_from_s3(key, allow_missing)
         if obj is None:
             return None