remove intermediate df

Author: vidhyamanisankar

statistical_methods_library/imputation/engine.py

@@ -579,7 +579,6 @@ def calculate_weighted_link(link_name):
     if manual_construction_col:
         # populate link, count, default information
         # for manual_construction data
-        unique_grp_prd = prepared_df.dropDuplicates(["period", "grouping"])
         # Get the required additional output columns
         mc_cols = manual_construction_df.columns
         mc_additional_cols = []
@@ -592,7 +591,11 @@ def calculate_weighted_link(link_name):
                 mc_additional_cols.append(key)
         manual_construction_df = (
             manual_construction_df.alias("mc")
-            .join(unique_grp_prd, ["period", "grouping"], "leftouter")
+            .join(
+                prepared_df.dropDuplicates(["period", "grouping"]),
+                ["period", "grouping"],
+                "leftouter",
+            )
             .select(
                 *(f"mc.{name}" for name in mc_cols),
                 *mc_additional_cols,