initial commit

statistical_methods_library/estimation.py

@@ -16,7 +16,7 @@ class ValidationError(Exception):
     pass
 
 
-def estimate(
+def ht_ratio(
     input_df: DataFrame,
     unique_identifier_col: str,
     period_col: str,

statistical_methods_library/imputation.py

@@ -56,7 +56,7 @@ class DataIntegrityError(ImputationError):
     pass
 
 
-def impute(
+def ratio_of_means(
     input_df: DataFrame,
     reference_col: str,
     period_col: str,

statistical_methods_library/winsorisation.py → statistical_methods_library/outliering.py

@@ -31,7 +31,7 @@ class Marker(Enum):
     """The value has not been winsorised because design * calibration is <= 1."""
 
 
-def one_sided_winsorise(
+def winsorisation(
     input_df: DataFrame,
     reference_col: str,
     period_col: str,

tests/test_estimation.py

@@ -80,7 +80,7 @@
 def test_input_not_a_dataframe():
     with pytest.raises(TypeError):
         # noinspection PyTypeChecker
-        estimation.estimate("not_a_dataframe", *params)
+        estimation.ht_ratio("not_a_dataframe", *params)
 
 
 # --- Test validation fail if mismatched death cols  ---
@@ -91,7 +91,7 @@ def test_params_mismatched_death_cols(fxt_load_test_csv):
     )
     bad_params = (unique_identifier_col, period_col, strata_col, sample_col, death_col)
     with pytest.raises(TypeError):
-        estimation.estimate(test_dataframe, *bad_params)
+        estimation.ht_ratio(test_dataframe, *bad_params)
 
 
 # --- Test validation fail if mismatched calibration cols  ---
@@ -108,7 +108,7 @@ def test_params_mismatched_calibration_cols(fxt_load_test_csv):
         calibration_group_col,
     )
     with pytest.raises(TypeError):
-        estimation.estimate(test_dataframe, *bad_params)
+        estimation.ht_ratio(test_dataframe, *bad_params)
 
 
 # --- Test if params not strings  ---
@@ -119,7 +119,7 @@ def test_params_not_string(fxt_load_test_csv):
     )
     bad_params = (unique_identifier_col, period_col, ["strata_col"], sample_col)
     with pytest.raises(TypeError):
-        estimation.estimate(test_dataframe, *bad_params)
+        estimation.ht_ratio(test_dataframe, *bad_params)
 
 
 # --- Test if params null  ---
@@ -130,7 +130,7 @@ def test_params_null(fxt_load_test_csv):
     )
     bad_params = (unique_identifier_col, period_col, "", sample_col)
     with pytest.raises(ValueError):
-        estimation.estimate(test_dataframe, *bad_params)
+        estimation.ht_ratio(test_dataframe, *bad_params)
 
 
 # --- Test validation fail if nulls in data  ---
@@ -140,7 +140,7 @@ def test_dataframe_nulls_in_data(fxt_load_test_csv):
         dataframe_columns, dataframe_types, "estimation", "unit", "null_value_present"
     )
     with pytest.raises(estimation.ValidationError):
-        estimation.estimate(test_dataframe, *params)
+        estimation.ht_ratio(test_dataframe, *params)
 
 
 # --- Test if cols missing from input dataframe(s)  ---
@@ -151,7 +151,7 @@ def test_dataframe_column_missing(fxt_load_test_csv):
     )
     bad_dataframe = test_dataframe.drop(strata_col)
     with pytest.raises(estimation.ValidationError):
-        estimation.estimate(bad_dataframe, *params)
+        estimation.ht_ratio(bad_dataframe, *params)
 
 
 # --- Test if references are duplicated in the input dataframe  ---
@@ -161,7 +161,7 @@ def test_dataframe_duplicate_reference(fxt_load_test_csv):
         dataframe_columns, dataframe_types, "estimation", "unit", "duplicate_references"
     )
     with pytest.raises(estimation.ValidationError):
-        estimation.estimate(test_dataframe, *params)
+        estimation.ht_ratio(test_dataframe, *params)
 
 
 # --- Test validation fail if non-boolean markers in data  ---
@@ -171,7 +171,7 @@ def test_dataframe_non_boolean_markers(fxt_load_test_csv):
         dataframe_columns, dataframe_types, "estimation", "unit", "non_boolean_markers"
     )
     with pytest.raises(estimation.ValidationError):
-        estimation.estimate(test_dataframe, *params)
+        estimation.ht_ratio(test_dataframe, *params)
 
 
 @pytest.mark.dependency()
@@ -185,7 +185,7 @@ def test_dataframe_large_death_count(fxt_load_test_csv):
     )
     with pytest.raises(estimation.ValidationError):
         estimation_params = [*params, death_col, h_col]
-        estimation.estimate(test_dataframe, *estimation_params)
+        estimation.ht_ratio(test_dataframe, *estimation_params)
 
 
 # --- Test validation fail if mixed h values in a strata  ---
@@ -200,7 +200,7 @@ def test_dataframe_mixed_h_values_in_strata(fxt_load_test_csv):
     )
     with pytest.raises(estimation.ValidationError):
         estimation_params = [*params, death_col, h_col]
-        estimation.estimate(test_dataframe, *estimation_params)
+        estimation.ht_ratio(test_dataframe, *estimation_params)
 
 
 # --- Test output is correct type ---
@@ -211,7 +211,7 @@ def test_dataframe_correct_type(fxt_spark_session, fxt_load_test_csv):
     )
 
     test_dataframe = test_dataframe.withColumn("bonus_column", lit(0))
-    ret_val = estimation.estimate(test_dataframe, *params)
+    ret_val = estimation.ht_ratio(test_dataframe, *params)
     assert isinstance(ret_val, type(test_dataframe))
 
 
@@ -222,7 +222,7 @@ def test_dataframe_no_extra_columns(fxt_spark_session, fxt_load_test_csv):
         dataframe_columns, dataframe_types, "estimation", "unit", "basic_functionality"
     )
     test_dataframe = test_dataframe.withColumn("bonus_column", lit(0))
-    ret_val = estimation.estimate(test_dataframe, *params)
+    ret_val = estimation.ht_ratio(test_dataframe, *params)
     # perform action on the dataframe to trigger lazy evaluation
     ret_val.count()
     ret_cols = ret_val.columns
@@ -235,7 +235,7 @@ def test_dataframe_expected_columns(fxt_spark_session, fxt_load_test_csv):
     test_dataframe = fxt_load_test_csv(
         dataframe_columns, dataframe_types, "estimation", "unit", "basic_functionality"
     )
-    ret_val = estimation.estimate(
+    ret_val = estimation.ht_ratio(
         test_dataframe,
         *params,
         auxiliary_col=auxiliary_col,
@@ -260,7 +260,7 @@ def test_dataframe_expected_columns_not_defaults(fxt_spark_session, fxt_load_tes
     test_dataframe = fxt_load_test_csv(
         dataframe_columns, dataframe_types, "estimation", "unit", "basic_functionality"
     )
-    ret_val = estimation.estimate(
+    ret_val = estimation.ht_ratio(
         test_dataframe,
         *params,
         auxiliary_col=auxiliary_col,
@@ -343,7 +343,7 @@ def test_calculations(fxt_load_test_csv, scenario_type, scenario):
         f"{scenario}_output",
     )
 
-    ret_val = estimation.estimate(test_dataframe, **estimation_kwargs)
+    ret_val = estimation.ht_ratio(test_dataframe, **estimation_kwargs)
 
     assert isinstance(ret_val, type(test_dataframe))
     sort_col_list = ["period", "strata"]

tests/test_imputation.py

@@ -140,7 +140,7 @@ def scenarios(categories):
 def test_input_not_a_dataframe():
     with pytest.raises(TypeError):
         # noinspection PyTypeChecker
-        imputation.impute("not_a_dataframe", *params)
+        imputation.ratio_of_means("not_a_dataframe", *params)
 
 
 # --- Test type validation on the back_data dataframe(s) ---
@@ -153,7 +153,9 @@ def test_back_data_not_a_dataframe(fxt_load_test_csv):
     )
     with pytest.raises(TypeError):
         # noinspection PyTypeChecker
-        imputation.impute(test_dataframe, *params, back_data_df="not_a_dataframe")
+        imputation.ratio_of_means(
+            test_dataframe, *params, back_data_df="not_a_dataframe"
+        )
 
 
 # --- Test if cols missing from input dataframe(s) ---
@@ -166,7 +168,7 @@ def test_dataframe_column_missing(fxt_load_test_csv):
     )
     bad_dataframe = test_dataframe.drop(strata_col)
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(bad_dataframe, *params)
+        imputation.ratio_of_means(bad_dataframe, *params)
 
 
 # --- Test if dataframe has duplicate rows ---
@@ -178,7 +180,7 @@ def test_dataframe_duplicate_rows(fxt_load_test_csv):
         dataframe_columns, dataframe_types, "imputation", "unit", "duplicate_rows"
     )
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params)
+        imputation.ratio_of_means(test_dataframe, *params)
 
 
 # --- Test if target missing from input dataframe(s) ---
@@ -191,7 +193,7 @@ def test_dataframe_target_missing(fxt_load_test_csv):
     )
     bad_dataframe = test_dataframe.drop(target_col)
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(bad_dataframe, *params)
+        imputation.ratio_of_means(bad_dataframe, *params)
 
 
 # --- Test if params null ---
@@ -212,7 +214,7 @@ def test_params_null(fxt_load_test_csv):
         marker_col,
     )
     with pytest.raises(ValueError):
-        imputation.impute(test_dataframe, *bad_params)
+        imputation.ratio_of_means(test_dataframe, *bad_params)
 
 
 @pytest.mark.dependency()
@@ -221,7 +223,7 @@ def test_params_missing_link_column(fxt_load_test_csv):
         dataframe_columns, dataframe_types, "imputation", "unit", "basic_functionality"
     )
     with pytest.raises(TypeError):
-        imputation.impute(
+        imputation.ratio_of_means(
             test_dataframe, *params, construction_link_col=construction_col
         )
 
@@ -241,7 +243,7 @@ def test_params_not_string(fxt_load_test_csv):
         marker_col,
     )
     with pytest.raises(TypeError):
-        imputation.impute(test_dataframe, *bad_params)
+        imputation.ratio_of_means(test_dataframe, *bad_params)
 
 
 # --- Test if output contents are as expected, both new columns and data ---
@@ -254,7 +256,7 @@ def test_dataframe_returned_as_expected(fxt_spark_session, fxt_load_test_csv):
     )
     # Make sure that no extra columns pass through.
     test_dataframe = test_dataframe.withColumn("bonus_column", lit(0))
-    ret_val = imputation.impute(test_dataframe, *params)
+    ret_val = imputation.ratio_of_means(test_dataframe, *params)
     # perform action on the dataframe to trigger lazy evaluation
     ret_val.count()
     assert isinstance(ret_val, type(test_dataframe))
@@ -276,7 +278,7 @@ def test_back_data_missing_column(fxt_load_test_csv, fxt_spark_session):
         "back_data_missing_column",
     )
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params, back_data_df=bad_back_data)
+        imputation.ratio_of_means(test_dataframe, *params, back_data_df=bad_back_data)
 
 
 @pytest.mark.dependency()
@@ -289,7 +291,7 @@ def test_back_data_contains_nulls(fxt_load_test_csv, fxt_spark_session):
     )
 
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params, back_data_df=bad_back_data)
+        imputation.ratio_of_means(test_dataframe, *params, back_data_df=bad_back_data)
 
 
 @pytest.mark.dependency()
@@ -306,7 +308,7 @@ def test_back_data_without_output_is_invalid(fxt_load_test_csv, fxt_spark_sessio
     )
 
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params, back_data_df=bad_back_data)
+        imputation.ratio_of_means(test_dataframe, *params, back_data_df=bad_back_data)
 
 
 # --- Test if when the back data input has link cols and the main data input does not
@@ -327,7 +329,7 @@ def test_back_data_drops_link_cols_when_present(fxt_load_test_csv, fxt_spark_ses
         "back_data_with_link_cols",
     )
 
-    ret_val = imputation.impute(test_dataframe, *params, back_data_df=back_data)
+    ret_val = imputation.ratio_of_means(test_dataframe, *params, back_data_df=back_data)
 
     assert ret_val.count() == 1
 
@@ -362,7 +364,7 @@ def test_input_has_link_cols_and_back_data_does_not_have_link_cols(
         "construction_link_col": construction_col,
     }
 
-    ret_val = imputation.impute(
+    ret_val = imputation.ratio_of_means(
         test_dataframe, *params, **imputation_kwargs, back_data_df=back_data
     )
 
@@ -382,7 +384,7 @@ def test_incorrect_column_types(fxt_load_test_csv):
         "basic_functionality",
     )
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params)
+        imputation.ratio_of_means(test_dataframe, *params)
 
 
 @pytest.mark.dependency()
@@ -392,7 +394,7 @@ def test_input_data_contains_nulls(fxt_load_test_csv, fxt_spark_session):
     )
 
     with pytest.raises(imputation.ValidationError):
-        imputation.impute(test_dataframe, *params)
+        imputation.ratio_of_means(test_dataframe, *params)
 
 
 # --- Test expected columns are in the output ---
@@ -401,7 +403,7 @@ def test_dataframe_expected_columns(fxt_spark_session, fxt_load_test_csv):
     test_dataframe = fxt_load_test_csv(
         dataframe_columns, dataframe_types, "imputation", "unit", "basic_functionality"
     )
-    ret_val = imputation.impute(
+    ret_val = imputation.ratio_of_means(
         test_dataframe,
         *default_params,
     )
@@ -477,7 +479,7 @@ def test_calculations(fxt_load_test_csv, scenario_type, scenario):
         f"{scenario}_output",
     )
 
-    ret_val = imputation.impute(test_dataframe, *params, **imputation_kwargs)
+    ret_val = imputation.ratio_of_means(test_dataframe, *params, **imputation_kwargs)
 
     select_cols = list(set(dataframe_columns) & set(exp_val.columns))
     assert isinstance(ret_val, type(test_dataframe))
@@ -582,7 +584,7 @@ def test_back_data_calculations(fxt_load_test_csv, scenario_type, scenario):
             "back_data_df": back_data_df.select(select_back_data_cols),
         }
 
-    ret_val = imputation.impute(input_df, *params, **imputation_kwargs)
+    ret_val = imputation.ratio_of_means(input_df, *params, **imputation_kwargs)
 
     assert isinstance(ret_val, type(test_dataframe))
     sort_col_list = ["reference", "period"]