better comparing to old language files

config_selector.py

@@ -19,6 +19,13 @@ BASE_DIR = os.getcwd()
 INGEST_DIR = os.path.join(BASE_DIR, "ingest")
 ORIGINAL_CONFIG_DIR = os.path.join(BASE_DIR, "original_config")
 RECIPE_CONFIG_DIR = os.path.join(BASE_DIR, "recipes", "wisski_default_data_model", "config")
+RECIPE_SOURCE_DIR = os.path.join(BASE_DIR, "recipes", "wisski_default_data_model")
+RECIPE_UPDATE_BASE = os.path.join(BASE_DIR, "update")
+
+
+def recipe_update_dir(recipe_source: str = RECIPE_SOURCE_DIR) -> str:
+    """Path to updated recipe copy: update/<recipe_name>/."""
+    return os.path.join(RECIPE_UPDATE_BASE, os.path.basename(recipe_source))
 
 
 def _clear_dir(path: str) -> int:
@@ -36,6 +43,13 @@ def _clear_dir(path: str) -> int:
     return removedCount
 
 
+def _copy_recipe_tree(src: str, dst: str) -> None:
+    """Copy a full recipe directory tree, including .git."""
+    if os.path.exists(dst):
+        shutil.rmtree(dst)
+    shutil.copytree(src, dst)
+
+
 def ingest_config(force: bool = False) -> None:
     """Extract a config*.tar.gz archive from ingest/ into original_config/.
 
@@ -103,6 +117,7 @@ class ConfigSelector:
         self.changed_files_path = changed_files_path or os.path.join(BASE_DIR, "changed_files")
         self.deleted_files_csv = deleted_files_csv or os.path.join(BASE_DIR, "deleted_files.csv")
         self.new_recipe_not_in_original_csv = os.path.join(BASE_DIR, "new_recipe_not_in_original.csv")
+        self.language_inconsistency_csv = os.path.join(BASE_DIR, "language_inconsistency.csv")
         self.changed_files_manifest_csv = os.path.join(BASE_DIR, "changed_files_manifest.csv")
         self.ignored_files_yml = ignored_files_yml or os.path.join(BASE_DIR, "ignored_files.yml")
         self.prefixes: List[str] = []
@@ -404,7 +419,107 @@ class ConfigSelector:
         """Join config root with a posix rel_path from _collect_yaml_relpaths."""
         return os.path.normpath(os.path.join(config_root, *rel_path.split("/")))
 
-    def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int]:
+    def _collect_language_inconsistencies(self) -> List[str]:
+        """Language overrides in config_path with no matching base *.yml at config root."""
+        inconsistent: List[str] = []
+        lang_root = os.path.join(self.config_path, "language")
+        if not os.path.isdir(lang_root):
+            return inconsistent
+        try:
+            for lang_code in os.listdir(lang_root):
+                lang_dir = os.path.join(lang_root, lang_code)
+                if not os.path.isdir(lang_dir):
+                    continue
+                for name in os.listdir(lang_dir):
+                    if not name.endswith(".yml"):
+                        continue
+                    base_path = os.path.join(self.config_path, name)
+                    if not os.path.isfile(base_path):
+                        inconsistent.append(f"language/{lang_code}/{name}")
+        except OSError:
+            pass
+        return sorted(inconsistent)
+
+    def _write_language_inconsistency_csv(self, inconsistent: List[str]) -> None:
+        with open(self.language_inconsistency_csv, "w", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["filename"])
+            for rel_path in inconsistent:
+                writer.writerow([rel_path])
+
+    def delete_language_inconsistencies(self) -> int:
+        """Remove language/*/*.yml in config_path that have no base config pendant."""
+        inconsistent = self._collect_language_inconsistencies()
+        removed = 0
+        for rel_path in inconsistent:
+            path = self._abs_config_path(self.config_path, rel_path)
+            try:
+                if os.path.isfile(path):
+                    os.remove(path)
+                    removed += 1
+            except OSError:
+                pass
+        self._remove_empty_language_dirs(self.config_path)
+        remaining = self._collect_language_inconsistencies()
+        self._write_language_inconsistency_csv(remaining)
+        return removed
+
+    @staticmethod
+    def _read_deleted_files_csv(csv_path: str) -> List[str]:
+        paths: List[str] = []
+        if not os.path.isfile(csv_path):
+            return paths
+        with open(csv_path, newline="") as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                name = (row.get("filename") or "").strip()
+                if name:
+                    paths.append(name)
+        return paths
+
+    def update_recipe(self) -> Tuple[int, int, int, str]:
+        """Copy committed recipe to update/<recipe_name>/, apply deletions and changed_files/.
+
+        Returns:
+            Tuple of (files_deleted, files_applied, deletes_missing, update_dir).
+        """
+        if not os.path.isdir(RECIPE_SOURCE_DIR):
+            raise FileNotFoundError(f"Recipe source not found: {RECIPE_SOURCE_DIR}")
+
+        update_dir = recipe_update_dir()
+        _copy_recipe_tree(RECIPE_SOURCE_DIR, update_dir)
+        update_config = os.path.join(update_dir, "config")
+
+        files_deleted = 0
+        deletes_missing = 0
+        for rel_path in self._read_deleted_files_csv(self.deleted_files_csv):
+            target = self._abs_config_path(update_config, rel_path)
+            try:
+                if os.path.isfile(target):
+                    os.remove(target)
+                    files_deleted += 1
+                else:
+                    deletes_missing += 1
+            except OSError:
+                deletes_missing += 1
+        self._remove_empty_language_dirs(update_config)
+
+        files_applied = 0
+        for rel_path in sorted(self._collect_yaml_relpaths(self.changed_files_path)):
+            src = self._abs_config_path(self.changed_files_path, rel_path)
+            dst = self._abs_config_path(update_config, rel_path)
+            os.makedirs(os.path.dirname(dst), exist_ok=True)
+            shutil.copy2(src, dst)
+            files_applied += 1
+
+        print(
+            f"Updated recipe written to {update_dir}/ "
+            f"({files_deleted} deleted from config, {files_applied} copied from changed_files/, "
+            f"{deletes_missing} delete entries not present in copy)."
+        )
+        return files_deleted, files_applied, deletes_missing, update_dir
+
+    def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int, int]:
         """
         Compare committed recipe config (old_recipe_path) to new_recipe_config (config_path).
 
@@ -414,10 +529,12 @@ class ConfigSelector:
         - Writes deleted_files.csv: paths in committed recipe missing from new_recipe_config.
         - Writes changed_files_manifest.csv: relative_path + kind (new|modified) for each copy.
         - Writes new_recipe_not_in_original.csv for paths in new_recipe not in original export.
+        - Writes language_inconsistency.csv for language/*/*.yml without a base config pendant.
 
         Returns:
             Tuple of (added_count, modified_count, deleted_count, unchanged_count,
-                      language_deleted_count, not_in_original_count)
+                      language_deleted_count, not_in_original_count,
+                      language_inconsistency_count)
         """
         _clear_dir(self.changed_files_path)
         os.makedirs(self.changed_files_path, exist_ok=True)
@@ -523,6 +640,16 @@ class ConfigSelector:
         elif not orig_paths:
             print("Warning: original_config has no YAML; orphan CSV is empty.")
 
+        language_inconsistent = self._collect_language_inconsistencies()
+        language_inconsistency_count = len(language_inconsistent)
+        self._write_language_inconsistency_csv(language_inconsistent)
+        if language_inconsistency_count:
+            print(
+                f"{language_inconsistency_count} language override(s) in new_recipe_config "
+                f"have no base config pendant "
+                f"(see {os.path.basename(self.language_inconsistency_csv)})."
+            )
+
         return (
             added_count,
             modified_count,
@@ -530,6 +657,7 @@ class ConfigSelector:
             unchanged_count,
             language_deleted_count,
             not_in_original_count,
+            language_inconsistency_count,
         )
 
     def add_prefix(self, prefix: str) -> None:
@@ -715,7 +843,11 @@ def draw_menu(stdscr, selector: ConfigSelector, selected_idx: int, view_mode: st
 
     # Instructions
     stdscr.addstr(h-4, 2, "LEFT/RIGHT: Navigate tree levels | SPACE: Toggle view | ENTER: Select files")
-    stdscr.addstr(h-3, 2, "A: Add prefix | O: Choose | D: Add/Del | C: Copy | M: Compare | S: Save | Q: Quit")
+    stdscr.addstr(
+        h - 3,
+        2,
+        "A: Add | O: Choose | D: Add/Del | C: Copy | M: Compare | I: Lang orphans | U: Update | S: Save | Q: Quit",
+    )
 
     # Content area
     start_y = 3
@@ -916,7 +1048,6 @@ def confirm_dialog(stdscr, message):
     dialog_win.addstr(3, 2, "Press Y to confirm, any other key to cancel")
     dialog_win.refresh()
 
-    # Get input
     key = stdscr.getch()
     return key == ord('y') or key == ord('Y')
 
@@ -1295,6 +1426,78 @@ def confirm_yaml_cleaning(stdscr, selector):
 
     return confirm_dialog(stdscr, "Copy all matched files to config directory and remove UUID and _core?")
 
+def _curses_addstr_clipped(win, y: int, x: int, text: str, max_width: int, attrs: int = 0) -> None:
+    """Write text inside a bordered curses window, clipping to fit."""
+    win_h, _ = win.getmaxyx()
+    if y <= 0 or y >= win_h - 1:
+        return
+    clipped = text[: max(0, max_width - x - 1)]
+    if not clipped:
+        return
+    try:
+        if attrs:
+            win.addstr(y, x, clipped, attrs)
+        else:
+            win.addstr(y, x, clipped)
+    except curses.error:
+        pass
+
+def show_info_dialog(stdscr, message: str) -> None:
+    """Show a simple message dialog."""
+    h, w = stdscr.getmaxyx()
+    if h < 5 or w < 20:
+        return
+    dialog_h = min(5, h - 2)
+    dialog_w = min(max(40, len(message) + 4), w - 4)
+    dialog_y = max(0, (h - dialog_h) // 2)
+    dialog_x = max(0, (w - dialog_w) // 2)
+    if dialog_h <= 0 or dialog_w <= 0:
+        return
+    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
+    dialog_win.box()
+    _curses_addstr_clipped(dialog_win, 1, 2, message, dialog_w)
+    _curses_addstr_clipped(dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w)
+    dialog_win.refresh()
+    stdscr.getch()
+
+def show_update_recipe_results(
+    stdscr,
+    files_deleted: int,
+    files_applied: int,
+    deletes_missing: int,
+    update_dir: str,
+) -> None:
+    """Show the results of building update/<recipe_name>/ from the committed recipe."""
+    h, w = stdscr.getmaxyx()
+    dialog_w = min(60, w - 4)
+    content_lines = [
+        f"Recipe copy: {update_dir}/",
+        f"Deleted from config: {files_deleted}",
+        f"Applied from changed_files/: {files_applied}",
+    ]
+    if deletes_missing:
+        content_lines.append(f"Delete CSV entries not in copy: {deletes_missing}")
+    content_start = 3
+    dialog_h = min(h - 2, max(12, content_start + len(content_lines) + 3))
+    dialog_y = max(0, (h - dialog_h) // 2)
+    dialog_x = max(0, (w - dialog_w) // 2)
+    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
+    dialog_win.box()
+    _curses_addstr_clipped(
+        dialog_win, 1, 2, "Recipe Update Results", dialog_w, curses.A_BOLD
+    )
+    max_footer_row = dialog_h - 3
+    for i, line in enumerate(content_lines):
+        row = content_start + i
+        if row > max_footer_row:
+            break
+        _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w)
+    _curses_addstr_clipped(
+        dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w
+    )
+    dialog_win.refresh()
+    stdscr.getch()
+
 def show_comparison_results(
     stdscr,
     added_count: int,
@@ -1303,59 +1506,65 @@ def show_comparison_results(
     unchanged_count: int,
     language_deleted_count: int = 0,
     not_in_original_count: int = 0,
+    language_inconsistency_count: int = 0,
 ):
     """Show the results of file comparison."""
     h, w = stdscr.getmaxyx()
-    dialog_w = 60
+    dialog_w = min(60, w - 4)
     total_out = added_count + modified_count
-    # Rows from 3: 5 stats + optional notes + blank + 2 CSV hints; footer at dialog_h - 2
-    lines = (
-        5
-        + (1 if not_in_original_count else 0)
-        + (1 if language_deleted_count else 0)
-        + 1
-        + 2
-    )
-    last_content_row = 2 + lines
-    dialog_h = min(h - 2, max(14, last_content_row + 3))
-    dialog_y = (h - dialog_h) // 2
-    dialog_x = (w - dialog_w) // 2
-
-    # Draw dialog box
-    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
-    dialog_win.box()
-    dialog_win.addstr(1, 2, "File Comparison Results", curses.A_BOLD)
-    row = 3
-    dialog_win.addstr(row, 4, f"New (not in committed recipe): {added_count}"[: dialog_w - 6])
-    row += 1
-    dialog_win.addstr(row, 4, f"Modified vs committed: {modified_count}"[: dialog_w - 6])
-    row += 1
-    dialog_win.addstr(row, 4, f"Total → changed_files/: {total_out}"[: dialog_w - 6])
-    row += 1
     deleted_line = f"Deleted (recipe, missing in new): {deleted_count}"
     if language_deleted_count:
         deleted_line += f" ({language_deleted_count} language/)"
-    dialog_win.addstr(row, 4, deleted_line[: dialog_w - 6])
-    row += 1
-    dialog_win.addstr(row, 4, f"Unchanged: {unchanged_count}"[: dialog_w - 6])
-    row += 1
+
+    content_lines = [
+        f"New (not in committed recipe): {added_count}",
+        f"Modified vs committed: {modified_count}",
+        f"Total -> changed_files/: {total_out}",
+        deleted_line,
+        f"Unchanged: {unchanged_count}",
+    ]
     if not_in_original_count:
-        dialog_win.addstr(
-            row,
-            4,
-            f"Not in original export: {not_in_original_count} (new_recipe_not_in_original.csv)"[
-                : dialog_w - 6
-            ],
+        content_lines.append(
+            f"Not in original export: {not_in_original_count} "
+            "(new_recipe_not_in_original.csv)"
         )
-        row += 1
     if language_deleted_count:
-        dialog_win.addstr(row, 4, "Language deletions: paths under language/… in deleted CSV")
-        row += 1
-    row += 1
-    dialog_win.addstr(row, 4, "Manifest → changed_files_manifest.csv")
-    row += 1
-    dialog_win.addstr(row, 4, "Deletions → deleted_files.csv")
-    dialog_win.addstr(dialog_h - 2, 2, "Press any key to continue...")
+        content_lines.append(
+            "Language deletions: paths under language/ in deleted CSV"
+        )
+    if language_inconsistency_count:
+        content_lines.append(
+            f"Language inconsistency: {language_inconsistency_count} "
+            "(language_inconsistency.csv)"
+        )
+    content_lines.extend(
+        [
+            "",
+            "Manifest -> changed_files_manifest.csv",
+            "Deletions -> deleted_files.csv",
+        ]
+    )
+
+    content_start = 3
+    # Leave one blank row between content and footer (footer at dialog_h - 2).
+    dialog_h = min(h - 2, max(14, content_start + len(content_lines) + 3))
+    dialog_y = max(0, (h - dialog_h) // 2)
+    dialog_x = max(0, (w - dialog_w) // 2)
+
+    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
+    dialog_win.box()
+    _curses_addstr_clipped(
+        dialog_win, 1, 2, "File Comparison Results", dialog_w, curses.A_BOLD
+    )
+    max_footer_row = dialog_h - 3
+    for i, line in enumerate(content_lines):
+        row = content_start + i
+        if row > max_footer_row:
+            break
+        _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w)
+    _curses_addstr_clipped(
+        dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w
+    )
     dialog_win.refresh()
 
     # Wait for key
@@ -1530,6 +1739,7 @@ def main(stdscr):
                     unchanged_count,
                     language_deleted_count,
                     not_in_original_count,
+                    language_inconsistency_count,
                 ) = selector.compare_and_track_changes()
                 show_comparison_results(
                     stdscr,
@@ -1539,7 +1749,40 @@ def main(stdscr):
                     unchanged_count,
                     language_deleted_count,
                     not_in_original_count,
+                    language_inconsistency_count,
                 )
+        elif key == ord('i') or key == ord('I'):
+            inconsistent = selector._collect_language_inconsistencies()
+            if not inconsistent:
+                show_info_dialog(stdscr, "No language inconsistencies in new_recipe_config.")
+            elif confirm_dialog(
+                stdscr,
+                f"Delete {len(inconsistent)} language file(s) without base config pendant?",
+            ):
+                removed = selector.delete_language_inconsistencies()
+                show_info_dialog(
+                    stdscr,
+                    f"Removed {removed} language inconsistency file(s) from new_recipe_config.",
+                )
+        elif key == ord('u') or key == ord('U'):
+            if confirm_dialog(
+                stdscr,
+                f"Copy recipe to {recipe_update_dir()}/, remove deleted_files.csv "
+                "entries, apply changed_files/ to config?",
+            ):
+                try:
+                    files_deleted, files_applied, deletes_missing, update_dir = (
+                        selector.update_recipe()
+                    )
+                    show_update_recipe_results(
+                        stdscr,
+                        files_deleted,
+                        files_applied,
+                        deletes_missing,
+                        update_dir,
+                    )
+                except FileNotFoundError as exc:
+                    show_info_dialog(stdscr, str(exc))
         elif key == ord('s') or key == ord('S'):
             selector.save_prefixes()
         elif key in (10, 13):  # ENTER key to select individual files