Rename transformed_column_name param in transform_spark()

Author: deliahu

cli/cmd/init.go

@@ -299,7 +299,7 @@ def create_estimator(run_config, model_config):
 #       arg2: FLOAT
 `,
 
-		"implementations/transformers/transformer.py": `def transform_spark(data, columns, args, transformed_column):
+		"implementations/transformers/transformer.py": `def transform_spark(data, columns, args, transformed_column_name):
     """Transform a column in a PySpark context.
 
     This function is optional (recommended for large-scale data processing).
@@ -314,18 +314,18 @@ def create_estimator(run_config, model_config):
         args: A dict with the same structure as the transformer's input args
             containing the runtime values of the args.
 
-        transformed_column: The name of the column containing the transformed
+        transformed_column_name: The name of the column containing the transformed
             data that is to be appended to the dataframe.
 
     Returns:
-        The original 'data' dataframe with an added column with the name of the
-        transformed_column arg containing the transformed data.
+        The original 'data' dataframe with an added column named <transformed_column_name>
+        which contains the transformed data.
     """
 
     ## Sample transform_spark implementation:
     #
     # return data.withColumn(
-    #     transformed_column, ((data[columns["num"]] - args["mean"]) / args["stddev"])
+    #     transformed_column_name, ((data[columns["num"]] - args["mean"]) / args["stddev"])
     # )
 
     pass

docs/applications/implementations/transformers.md

@@ -5,7 +5,7 @@ Transformers run both when transforming data before model training and when resp
 ## Implementation
 
 ```python
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     """Transform a column in a PySpark context.
 
     This function is optional (recommended for large-scale data processing).
@@ -20,12 +20,12 @@ def transform_spark(data, columns, args, transformed_column):
         args: A dict with the same structure as the transformer's input args
             containing the runtime values of the args.
 
-        transformed_column: The name of the column containing the transformed
+        transformed_column_name: The name of the column containing the transformed
             data that is to be appended to the dataframe.
 
     Returns:
-        The original 'data' dataframe with an added column with the name of the
-        transformed_column arg containing the transformed data.
+        The original 'data' dataframe with an added column named <transformed_column_name>
+        which contains the transformed data.
     """
     pass
 
@@ -69,9 +69,9 @@ def reverse_transform_python(transformed_value, args):
 ## Example
 
 ```python
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     return data.withColumn(
-        transformed_column, ((data[columns["num"]] - args["mean"]) / args["stddev"])
+        transformed_column_name, ((data[columns["num"]] - args["mean"]) / args["stddev"])
     )
 
 def transform_python(sample, args):

examples/fraud/implementations/transformers/weight.py

@@ -1,9 +1,9 @@
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     import pyspark.sql.functions as F
 
     distribution = args["class_distribution"]
 
     return data.withColumn(
-        transformed_column,
+        transformed_column_name,
         F.when(data[columns["col"]] == 0, distribution[1]).otherwise(distribution[0]),
     )

pkg/transformers/bucketize.py

@@ -13,15 +13,15 @@
 # limitations under the License.
 
 
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     from pyspark.ml.feature import Bucketizer
     import pyspark.sql.functions as F
 
     new_b = Bucketizer(
-        splits=args["bucket_boundaries"], inputCol=columns["num"], outputCol=transformed_column
+        splits=args["bucket_boundaries"], inputCol=columns["num"], outputCol=transformed_column_name
     )
     return new_b.transform(data).withColumn(
-        transformed_column, F.col(transformed_column).cast("int")
+        transformed_column_name, F.col(transformed_column_name).cast("int")
     )
 
 

pkg/transformers/index_string.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 
 
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     from pyspark.ml.feature import StringIndexerModel
     import pyspark.sql.functions as F
 
     indexer = StringIndexerModel.from_labels(
-        args["index"], inputCol=columns["text"], outputCol=transformed_column
+        args["index"], inputCol=columns["text"], outputCol=transformed_column_name
     )
 
     return indexer.transform(data).withColumn(
-        transformed_column, F.col(transformed_column).cast("int")
+        transformed_column_name, F.col(transformed_column_name).cast("int")
     )
 
 

pkg/transformers/normalize.py

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 
-def transform_spark(data, columns, args, transformed_column):
+def transform_spark(data, columns, args, transformed_column_name):
     return data.withColumn(
-        transformed_column, ((data[columns["num"]] - args["mean"]) / args["stddev"])
+        transformed_column_name, ((data[columns["num"]] - args["mean"]) / args["stddev"])
     )
 
 

pkg/workloads/lib/context.py

@@ -481,7 +481,7 @@ def resource_status_key(self, resource):
 
 TRANSFORMER_IMPL_VALIDATION = {
     "optional": [
-        {"name": "transform_spark", "args": ["data", "columns", "args", "transformed_column"]},
+        {"name": "transform_spark", "args": ["data", "columns", "args", "transformed_column_name"]},
         {"name": "reverse_transform_python", "args": ["transformed_value", "args"]},
         {"name": "transform_python", "args": ["sample", "args"]},
     ]