frederikhoengaard
diff --git a/‎README.md
+21-1 b/‎README.md
+21-1
diff --git a/‎python/src/lazylearn/lazylearn.py
+5-2 b/‎python/src/lazylearn/lazylearn.py
+5-2
diff --git a/‎python/src/lazylearn/regression/models/xgboost/xgb_regressor_steps/hpo_step.py
+2-2 b/‎python/src/lazylearn/regression/models/xgboost/xgb_regressor_steps/hpo_step.py
+2-2
diff --git a/‎python/src/lazylearn/strategies/strategy_builder.py
+3-2 b/‎python/src/lazylearn/strategies/strategy_builder.py
+3-2
diff --git a/‎python/src/lazylearn/strategies/strategy_steps/__init__.py b/‎python/src/lazylearn/strategies/strategy_steps/__init__.py
diff --git a/‎python/src/lazylearn/strategies/strategy_steps/evaluation.py
+27 b/‎python/src/lazylearn/strategies/strategy_steps/evaluation.py
+27
@@ -19,7 +19,27 @@ Current stable version is 0.0.3. The upcoming updates will support:
 
 ## Usage
 
-Using lazy-learn revolves around the `LazyLearner` class. You can think of it as a kind of project, and it is the wrapper for any experiment within lazy-learn.
+Using lazy-learn revolves around the `LazyLearner` class. You can think of it as a kind of project, and it is the wrapper for any experiment within lazy-learn. You can consider a simple example with the California Housing dataset:
+
+```python
+from lazylearn import LazyLearner
+from sklearn.datasets import fetch_california_housing
+
+
+# get some data
+data = fetch_california_housing(as_frame=True)
+df = data["data"]
+df["MedHouseVal"] = data["target"]
+
+# instantiate and run the LazyLearner
+learner = LazyLearner()
+learner.create_project(data=df, target="MedHouseVal")
+learner.run_autopilot()
+
+# evaluate results
+print(learner.leaderboard())
+
+```
 
 ## Installation
 
 
@@ -16,8 +16,9 @@ def __init__(self, random_state=None):
         self._leaderboard = None
         self.random_state = random_state
         self.target = None
+        self.metric = None
 
-    def create_project(self, data, target, task="infer"):
+    def create_project(self, data, target, task="infer", metric="default"):
         # ingest data
         self.target = target
         self.dataset = Ingestion().run(data)
@@ -26,6 +27,8 @@ def create_project(self, data, target, task="infer"):
             # if target is numeric then regression, else classification
             if self.dataset.column_type_map[target] == "numeric":
                 self.task = "regression"
+                if metric == "default":
+                    self.metric = "mse"
             else:
                 self.task = "classification"
 
@@ -57,7 +60,7 @@ def run_autopilot(self):
             random_state=self.random_state,
         )
         self._leaderboard = sorted(
-            [model for model in sb.models], key=lambda x: x.score
+            [model for model in sb.models], key=lambda x: x.score[self.metric]
         )
 
     def leaderboard(self):
 
@@ -9,13 +9,13 @@ def __init__(self, n_splits=5, random_state=None):
         self.random_state = random_state
         self.param_grid = {
             "max_depth": [3, 4, 5, 6, 7, 8, 9, 10],
-            "learning_rate": [0.001, 0.01, 0.1, 0.2, 0.3],
+            "learning_rate": [0.001, 0.01, 0.1, 0.2, 0.3, 0.5],
             "subsample": [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
             "colsample_bytree": [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
             "colsample_bylevel": [0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
             "min_child_weight": [0.5, 1.0, 3.0, 5.0, 7.0, 10.0],
             "gamma": [0, 0.25, 0.5, 1.0],
-            "n_estimators": [100, 200, 300, 500, 1000],
+            "n_estimators": [100, 200, 300, 500, 1000, 2500],
         }
 
     def fit(self, pipeline: RegressionPipeline):
 
@@ -3,7 +3,7 @@
     RandomForestRegressionRunner,
 )
 from regression.models.xgboost.xgb import XGBRegressionRunner
-from sklearn.metrics import mean_absolute_error
+from strategies.strategy_steps.evaluation import Evaluator
 
 
 class StrategyBuilder:
@@ -44,7 +44,8 @@ def start(self):
 
             # get holdout scores
             strategy.predict(self.dataset.partitions["test"].copy())
-            strategy.pipeline.holdout_score = mean_absolute_error(
+            strategy.pipeline.holdout_score = Evaluator().evaluate(
+                self.task,
                 self.dataset.partitions["test"][self.target],
                 strategy.pipeline.tmp_pred,
             )
 
@@ -0,0 +1,27 @@
+from sklearn.metrics import (
+    accuracy_score,
+    f1_score,
+    log_loss,
+    mean_absolute_error,
+    mean_absolute_percentage_error,
+    mean_squared_error,
+)
+
+
+class Evaluator:
+    def __init__(self):
+        self.metrics = {
+            "regression": [
+                ("mae", mean_absolute_error),
+                ("mse", mean_squared_error),
+                ("mape", mean_absolute_percentage_error),
+            ],
+            "classification": [
+                ("accuracy", accuracy_score),
+                ("f1", f1_score),
+                ("logloss", log_loss),
+            ],
+        }
+
+    def evaluate(self, task, y_pred, y_true):
+        return {name: func(y_true, y_pred) for name, func in self.metrics[task]}