WIP: Introduce jobs for more flexible task distribution

mlonmcu/cli/common.py

@@ -201,23 +201,10 @@ def kickoff_runs(args, until, context):
     assert len(context.sessions) > 0
     session = context.sessions[-1]
     # session.label = args.label
-    config = extract_config(args)
-    # TODO: move into context/session
-    per_stage = True
-    print_report = True
-    if "runs_per_stage" in config:
-        per_stage = bool(config["runs_per_stage"])
-    elif "runs_per_stage" in context.environment.vars:
-        per_stage = bool(context.environment.vars["runs_per_stage"])
-    if "print_report" in config:
-        print_report = bool(config["print_report"])
-    elif "print_report" in context.environment.vars:
-        print_report = bool(context.environment.vars["print_report"])
     success = session.process_runs(
         until=until,
-        per_stage=per_stage,
-        print_report=print_report,
         num_workers=args.parallel,
+        print_report=True,
         progress=args.progress,
         context=context,
         export=True,

mlonmcu/context.py

@@ -129,7 +129,7 @@ def get_environment_by_name(name: str) -> Environment:
     return None
 
 
-def get_ids(directory: Path) -> List[int]:
+def get_infos(directory: Path) -> List[tuple]:
     """Get a sorted list of ids for sessions/runs found in the given directory.
 
     Parameters
@@ -139,13 +139,26 @@ def get_ids(directory: Path) -> List[int]:
 
     Returns:
     list
-        List of integers representing the session numbers. Empty list if directory does not exist.
+        List of tuples containing the session/run numbers and their hidden attribute.
+        Empty list if directory does not exist.
     """
     if not directory.is_dir():
         return []
 
-    ids = [int(o) for o in os.listdir(directory) if os.path.isdir(directory / o) and not os.path.islink(directory / o)]
-    return sorted(ids)  # TODO: sort by session datetime?
+    def extract_idx(x):
+        hidden = False
+        if x[0] == ".":
+            hidden = True
+            x = x[1:]
+        if "_" in x:
+            idx = x.split("_")
+            idx = tuple(list(map(int, idx)))
+        else:
+            idx = int(x)
+        return idx, hidden
+
+    ids = [extract_idx(o) for o in os.listdir(directory) if os.path.isdir(directory / o) and not os.path.islink(directory / o)]
+    return sorted(ids, key=lambda x: x[0] if isinstance(x[0], tuple) else (x[0], ))  # TODO: sort by session datetime?
 
 
 def load_recent_sessions(env: Environment, count: int = None) -> List[Session]:
@@ -171,22 +184,23 @@ def load_recent_sessions(env: Environment, count: int = None) -> List[Session]:
     sessions_directory = env.paths["temp"].path / "sessions"
 
     # TODO: in the future also strs (custom or hash) should be allowed
-    session_ids = get_ids(sessions_directory)
+    session_infos = get_infos(sessions_directory)
 
-    for sid in session_ids:
+    for sid, _ in session_infos:
         session_directory = sessions_directory / str(sid)
         # session_file = sessions_directory / str(sid) / "session.txt"
         # if not session_file.is_file():
         #     continue
         runs_directory = session_directory / "runs"
-        run_ids = get_ids(runs_directory)
+        run_infos = get_infos(runs_directory)
         runs = []
-        for rid in run_ids:
+        for rid, hidden in run_infos:
             run_directory = runs_directory / str(rid)
             # run_file = run_directory / "run.txt"
             # run = Run.from_file(run_file)  # TODO: actually implement run restore
             run = Run()  # TODO: fix
             run.archived = True
+            run.hidden = hidden
             run.dir = run_directory
             runs.append(run)
         session = Session(idx=sid, archived=True, dir=session_directory)

mlonmcu/feature/feature.py

@@ -55,7 +55,12 @@ def helper(key):
         return {helper(key): value for key, value in config.items() if f"{self.name}." in key}
 
     def __repr__(self):
-        return type(self).__name__ + f"({self.name})"
+        probs = []
+        if self.name:
+            probs.append(self.name)
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Feature(" + ",".join(probs) + ")"
 
     # @property
     # def types(self):

mlonmcu/flow/backend.py

@@ -54,8 +54,14 @@ def __init__(
         self.tuner = None
 
     def __repr__(self):
-        name = type(self).name
-        return f"Backend({name})"
+        probs = []
+        if self.name:
+            probs.append(type(self).name)
+        if self.features and len(self.features) > 0:
+            probs.append(str(self.features))
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Backend(" + ",".join(probs) + ")"
 
     def process_features(self, features):
         # Filter out non-backend features

mlonmcu/flow/framework.py

@@ -38,6 +38,16 @@ def __init__(self, features=None, config=None, backends={}):
         self.features = self.process_features(features)
         self.config = filter_config(self.config, self.name, self.DEFAULTS, self.OPTIONAL, self.REQUIRED)
 
+    def __repr__(self):
+        probs = []
+        if self.name:
+            probs.append(type(self).name)
+        if self.features and len(self.features) > 0:
+            probs.append(str(self.features))
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Framework(" + ",".join(probs) + ")"
+
     def process_features(self, features):
         if features is None:
             return []

mlonmcu/models/model.py

@@ -99,6 +99,9 @@ def outputs_path(self):
         return self.config["outputs_path"]
 
     def __repr__(self):
-        if self.alt:
-            return f"Model({self.name},alt={self.alt})"
-        return f"Model({self.name})"
+        probs = []
+        if self.name:
+            probs.append(self.name)
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Model(" + ",".join(probs) + ")"

mlonmcu/platform/platform.py

@@ -67,6 +67,16 @@ def __init__(self, name, features=None, config=None):
         self.config = filter_config(self.config, self.name, self.DEFAULTS, self.OPTIONAL, self.REQUIRED)
         self.artifacts = []
 
+    def __repr__(self):
+        probs = []
+        if self.name:
+            probs.append(self.name)
+        if self.features and len(self.features) > 0:
+            probs.append(str(self.features))
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Platform(" + ",".join(probs) + ")"
+
     def init_directory(self, path=None, context=None):
         raise NotImplementedError
 

mlonmcu/session/group.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""TODO"""

mlonmcu/session/job.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""TODO"""

mlonmcu/session/postprocess/postprocess.py

@@ -36,6 +36,16 @@ def __init__(self, name, config=None, features=None):
         self.features = self.process_features(features)
         self.config = filter_config(self.config, self.name, self.DEFAULTS, self.OPTIONAL, self.REQUIRED)
 
+    def __repr__(self):
+        probs = []
+        if self.name:
+            probs.append(self.name)
+        if self.features and len(self.features) > 0:
+            probs.append(str(self.features))
+        if self.config and len(self.config) > 0:
+            probs.append(str(self.config))
+        return "Postprocess(" + ",".join(probs) + ")"
+
     def process_features(self, features):
         """Utility which handles postprocess_features."""
         # Currently there is no support for postprocess features (FIXME)

mlonmcu/session/progress.py

@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+#
+# This file is part of MLonMCU.
+# See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""TODO"""
+
+from tqdm import tqdm
+
+
+def init_progress(total, msg="Processing..."):
+    """Helper function to initialize a progress bar for the session."""
+    return tqdm(
+        total=total,
+        desc=msg,
+        ncols=100,
+        bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}s]",
+        leave=None,
+        mininterval=0.001,
+        # maxinterval=0,
+    )
+
+
+def update_progress(pbar, count=1):
+    """Helper function to update the progress bar for the session."""
+    pbar.update(count)
+
+
+def get_pbar_callback(pbar):
+    def callback(_):
+        update_progress(pbar)
+    return callback
+
+
+def close_progress(pbar):
+    """Helper function to close the session progressbar, if available."""
+    if pbar:
+        pbar.close()