better change file handling

config_selector.py

@@ -1,18 +1,97 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 import json
 import shutil
 import curses
 import re
 import csv
 import filecmp
+import glob
+import tarfile
 import yaml
 from typing import List, Dict, Set, Tuple, Optional
 
-# Use current directory instead of hardcoded path
+# Use current directory instead of hardcoded path.
 BASE_DIR = os.getcwd()
 
+INGEST_DIR = os.path.join(BASE_DIR, "ingest")
+ORIGINAL_CONFIG_DIR = os.path.join(BASE_DIR, "original_config")
+RECIPE_CONFIG_DIR = os.path.join(BASE_DIR, "recipes", "wisski_default_data_model", "config")
+
+
+def _clear_dir(path: str) -> int:
+    """Remove all files and subdirectories inside path. Returns count of removed items."""
+    if not os.path.isdir(path):
+        return 0
+    removedCount = 0
+    for entry in os.listdir(path):
+        entryPath = os.path.join(path, entry)
+        if os.path.isdir(entryPath):
+            shutil.rmtree(entryPath)
+        else:
+            os.remove(entryPath)
+        removedCount += 1
+    return removedCount
+
+
+def ingest_config(force: bool = False) -> None:
+    """Extract a config*.tar.gz archive from ingest/ into original_config/.
+
+    Skips extraction when YAML files are already present in original_config/,
+    unless force is True. With --force, original_config/, new_recipe_config/,
+    and changed_files/ are all cleared before re-extraction.
+    """
+    os.makedirs(ORIGINAL_CONFIG_DIR, exist_ok=True)
+
+    existingFiles = [f for f in os.listdir(ORIGINAL_CONFIG_DIR) if f.endswith(".yml")]
+    if existingFiles and not force:
+        print(
+            f"original_config/ already contains {len(existingFiles)} YAML files. "
+            "Use --force to re-extract."
+        )
+        return
+
+    archives = sorted(glob.glob(os.path.join(INGEST_DIR, "config*.tar.gz")))
+    if not archives:
+        if not existingFiles:
+            print("No config*.tar.gz found in ingest/ and original_config/ is empty.")
+        return
+
+    # Use the most recently named archive.
+    archive = archives[-1]
+    print(f"Extracting {os.path.basename(archive)} to original_config/...")
+
+    if force:
+        _clear_dir(ORIGINAL_CONFIG_DIR)
+        newRecipeDir = os.path.join(BASE_DIR, "new_recipe_config")
+        changedFilesDir = os.path.join(BASE_DIR, "changed_files")
+        cleared = _clear_dir(newRecipeDir)
+        if cleared:
+            print(f"Cleared {cleared} items from new_recipe_config/.")
+        cleared = _clear_dir(changedFilesDir)
+        if cleared:
+            print(f"Cleared {cleared} items from changed_files/.")
+
+    extractedCount = 0
+    with tarfile.open(archive, "r:gz") as tar:
+        for member in tar.getmembers():
+            if not (member.isfile() and member.name.endswith(".yml")):
+                continue
+            # Preserve subdirectory structure (e.g. language/de/foo.yml) but strip
+            # any leading archive root component that is not part of the config layout.
+            relPath = member.name.lstrip("./")
+            destPath = os.path.join(ORIGINAL_CONFIG_DIR, relPath)
+            os.makedirs(os.path.dirname(destPath), exist_ok=True)
+            srcFile = tar.extractfile(member)
+            if srcFile is not None:
+                with open(destPath, "wb") as dst:
+                    dst.write(srcFile.read())
+                extractedCount += 1
+
+    print(f"Extracted {extractedCount} YAML files to original_config/.")
+
 class ConfigSelector:
     def __init__(self, json_path: str, original_config_path: str, config_path: str,
                  old_recipe_path: str = None, changed_files_path: str = None, deleted_files_csv: str = None,
@@ -23,6 +102,8 @@ class ConfigSelector:
         self.old_recipe_path = old_recipe_path or os.path.join(BASE_DIR, "old_recipe_config")
         self.changed_files_path = changed_files_path or os.path.join(BASE_DIR, "changed_files")
         self.deleted_files_csv = deleted_files_csv or os.path.join(BASE_DIR, "deleted_files.csv")
+        self.new_recipe_not_in_original_csv = os.path.join(BASE_DIR, "new_recipe_not_in_original.csv")
+        self.changed_files_manifest_csv = os.path.join(BASE_DIR, "changed_files_manifest.csv")
         self.ignored_files_yml = ignored_files_yml or os.path.join(BASE_DIR, "ignored_files.yml")
         self.prefixes: List[str] = []
         self.matched_files: Dict[str, List[str]] = {}
@@ -61,7 +142,13 @@ class ConfigSelector:
                     # Iterate through all categories in the YAML.
                     for category, info in data.items():
                         if isinstance(info, dict) and 'name' in info:
-                            self.ignored_files.add(info['name'])
+                            name = info['name']
+                            if not isinstance(name, str):
+                                continue
+                            self.ignored_files.add(name)
+                            # Basenames on disk always include .yml; allow omitting it in YAML.
+                            if not name.endswith('.yml'):
+                                self.ignored_files.add(f'{name}.yml')
         except (yaml.YAMLError, FileNotFoundError) as e:
             print(f"Warning: Error loading ignored files YAML: {e}")
 
@@ -107,101 +194,291 @@ class ConfigSelector:
             print(f"Error: Directory {self.original_config_path} not found")
             exit(1)
 
+    @staticmethod
+    def _strip_metadata(content: str) -> str:
+        """Remove uuid and _core keys from Drupal YAML config content."""
+        content = re.sub(r'^uuid: [a-f0-9\-]+\n', '', content, flags=re.MULTILINE)
+        # Match exactly '_core:' at the start of a line (avoids touching wisski_core).
+        content = re.sub(r'^_core:\n([ \t]+[^\n]+\n)+', '', content, flags=re.MULTILINE)
+        return content
+
+    def _collect_bundle_hashes(self) -> Set[str]:
+        """Return the set of bundle hashes found in matched wisski_core.wisski_bundle files."""
+        bundlePrefix = "wisski_core.wisski_bundle."
+        bundleHashes: Set[str] = set()
+        for prefix in self.prefixes:
+            for filename in self.matched_files.get(prefix, []):
+                if filename.startswith(bundlePrefix) and filename.endswith(".yml"):
+                    bundleHash = filename[len(bundlePrefix):-4]
+                    bundleHashes.add(bundleHash)
+        return bundleHashes
+
+    def _copy_file(self, src: str, dst: str) -> None:
+        """Read src, strip metadata, write to dst. Falls back to raw copy on error."""
+        try:
+            with open(src, 'r') as f:
+                content = f.read()
+            content = self._strip_metadata(content)
+            with open(dst, 'w') as f:
+                f.write(content)
+        except Exception as e:
+            print(f"Error processing {os.path.basename(src)}: {e}")
+            shutil.copy2(src, dst)
+
+    def _copy_with_language_overrides(self, filename: str) -> int:
+        """Copy original_config/language/{lang}/{filename} to new_recipe_config/language/{lang}/
+        for every language that has an override for this file. Returns count of overrides copied."""
+        langBase = os.path.join(self.original_config_path, "language")
+        if not os.path.isdir(langBase):
+            return 0
+        overridesCopied = 0
+        for langCode in os.listdir(langBase):
+            srcLangDir = os.path.join(langBase, langCode)
+            if not os.path.isdir(srcLangDir):
+                continue
+            srcOverride = os.path.join(srcLangDir, filename)
+            if not os.path.isfile(srcOverride):
+                continue
+            dstLangDir = os.path.join(self.config_path, "language", langCode)
+            os.makedirs(dstLangDir, exist_ok=True)
+            self._copy_file(srcOverride, os.path.join(dstLangDir, filename))
+            overridesCopied += 1
+        return overridesCopied
+
     def copy_matched_files(self) -> None:
         """Copy all matched files from original_config to config, removing UUID and _core structures."""
         if not os.path.exists(self.config_path):
             os.makedirs(self.config_path)
 
-        copied_count = 0
-        processed_count = 0
+        copiedCount = 0
+        processedCount = 0
+        overrideCopied = 0
         for prefix in self.prefixes:
             for filename in self.matched_files[prefix]:
                 src = os.path.join(self.original_config_path, filename)
                 dst = os.path.join(self.config_path, filename)
-
-                # Read the YAML file
                 try:
-                    with open(src, 'r') as file:
-                        content = file.read()
-
-                    # Process the content to remove UUID and _core structures
-                    # 1. Remove uuid key-value pair (ensuring we start at beginning of line)
-                    content = re.sub(r'^uuid: [a-f0-9\-]+\n', '', content, flags=re.MULTILINE)
-
-                    # 2. Remove _core structure (careful not to remove wisski_core or similar)
-                    # Match exactly '_core:' at the start of a line
-                    content = re.sub(r'^_core:\n([ \t]+[^\n]+\n)+', '', content, flags=re.MULTILINE)
-
-                    # Write the modified content
-                    with open(dst, 'w') as file:
-                        file.write(content)
-
-                    processed_count += 1
+                    with open(src, 'r') as f:
+                        content = f.read()
+                    content = self._strip_metadata(content)
+                    with open(dst, 'w') as f:
+                        f.write(content)
+                    processedCount += 1
                 except Exception as e:
                     print(f"Error processing {filename}: {e}")
-                    # Fall back to direct copy if processing fails
                     shutil.copy2(src, dst)
+                copiedCount += 1
+                overrideCopied += self._copy_with_language_overrides(filename)
 
-                copied_count += 1
+        # For every matched bundle, also copy its language.content_settings counterpart.
+        langCopied = 0
+        for bundleHash in self._collect_bundle_hashes():
+            langFilename = f"language.content_settings.wisski_individual.{bundleHash}.yml"
+            langSrc = os.path.join(self.original_config_path, langFilename)
+            langDst = os.path.join(self.config_path, langFilename)
+            if os.path.exists(langSrc):
+                self._copy_file(langSrc, langDst)
+                langCopied += 1
+                overrideCopied += self._copy_with_language_overrides(langFilename)
 
-        print(f"Copied {copied_count} files to {self.config_path} ({processed_count} files were processed to remove UUIDs and _core structures)")
+        print(
+            f"Copied {copiedCount} files to {self.config_path} "
+            f"({processedCount} processed to remove UUIDs and _core structures)"
+        )
+        if langCopied:
+            print(f"Also copied {langCopied} corresponding language.content_settings files.")
+        if overrideCopied:
+            print(f"Also copied {overrideCopied} language override files from original_config/language/.")
+        pruned_export = self._prune_new_recipe_not_in_original()
+        if pruned_export:
+            print(
+                f"Removed {pruned_export} file(s) from new_recipe_config that are not in original_config "
+                f"(stale vs current export)."
+            )
+        pruned = self._prune_stale_language_overrides()
+        if pruned:
+            print(f"Removed {pruned} stale language override file(s) (no longer in export or missing base config).")
 
-    def compare_and_track_changes(self) -> Tuple[int, int, int]:
+    @staticmethod
+    def _remove_empty_language_dirs(config_root: str) -> None:
+        """Remove empty language/<lang>/ and language/ under config_root if possible."""
+        lang_root = os.path.join(config_root, "language")
+        if not os.path.isdir(lang_root):
+            return
+        for lang_code in list(os.listdir(lang_root)):
+            lang_dir = os.path.join(lang_root, lang_code)
+            if not os.path.isdir(lang_dir):
+                continue
+            try:
+                if not os.listdir(lang_dir):
+                    os.rmdir(lang_dir)
+            except OSError:
+                pass
+        try:
+            if not os.listdir(lang_root):
+                os.rmdir(lang_root)
+        except OSError:
+            pass
+
+    def _prune_new_recipe_not_in_original(self) -> int:
+        """Delete YAML under config_path whose relative path is absent from original_config.
+
+        Skips when original_config has no YAML (avoid wiping new_recipe before an export).
         """
-        Compare files between old_recipe_config and new_recipe_config.
-        Copy changed files to changed_files folder.
-        Track deleted files in deleted_files.csv.
+        orig_paths = self._collect_yaml_relpaths(self.original_config_path)
+        if not orig_paths:
+            return 0
+        new_paths = self._collect_yaml_relpaths(self.config_path)
+        orphans = new_paths - orig_paths
+        removed = 0
+        for rel in sorted(orphans):
+            path = self._abs_config_path(self.config_path, rel)
+            try:
+                if os.path.isfile(path):
+                    os.remove(path)
+                    removed += 1
+            except OSError:
+                pass
+        self._remove_empty_language_dirs(self.config_path)
+        return removed
+
+    def _prune_stale_language_overrides(self) -> int:
+        """Remove language/*/*.yml in config_path that should not be kept.
+
+        Drops overrides that are missing from the current export (translation removed
+        on the site) or whose base config is not present under config_path (orphans
+        from earlier copies). Without this, compare would not list those as deleted.
+        """
+        removed = 0
+        lang_dst_root = os.path.join(self.config_path, "language")
+        if not os.path.isdir(lang_dst_root):
+            return 0
+        lang_src_root = os.path.join(self.original_config_path, "language")
+        for lang_code in list(os.listdir(lang_dst_root)):
+            dst_lang_dir = os.path.join(lang_dst_root, lang_code)
+            if not os.path.isdir(dst_lang_dir):
+                continue
+            src_lang_dir = os.path.join(lang_src_root, lang_code)
+            for name in list(os.listdir(dst_lang_dir)):
+                if not name.endswith(".yml"):
+                    continue
+                base_fp = os.path.join(self.config_path, name)
+                src_fp = os.path.join(src_lang_dir, name)
+                if os.path.isfile(base_fp) and os.path.isfile(src_fp):
+                    continue
+                try:
+                    os.remove(os.path.join(dst_lang_dir, name))
+                    removed += 1
+                except OSError:
+                    pass
+        self._remove_empty_language_dirs(self.config_path)
+        return removed
+
+    @staticmethod
+    def _collect_yaml_relpaths(config_root: str) -> Set[str]:
+        """Relative paths (posix-style) for YAML under a Drupal config tree.
+
+        Includes top-level *.yml and language/<lang>/*.yml (same layout as exports).
+        """
+        rel_paths: Set[str] = set()
+        if not os.path.isdir(config_root):
+            return rel_paths
+        try:
+            for name in os.listdir(config_root):
+                if name.endswith(".yml"):
+                    rel_paths.add(name)
+            lang_root = os.path.join(config_root, "language")
+            if os.path.isdir(lang_root):
+                for lang_code in os.listdir(lang_root):
+                    lang_dir = os.path.join(lang_root, lang_code)
+                    if not os.path.isdir(lang_dir):
+                        continue
+                    for name in os.listdir(lang_dir):
+                        if name.endswith(".yml"):
+                            rel_paths.add(f"language/{lang_code}/{name}")
+        except OSError:
+            pass
+        return rel_paths
+
+    def _abs_config_path(self, config_root: str, rel_path: str) -> str:
+        """Join config root with a posix rel_path from _collect_yaml_relpaths."""
+        return os.path.normpath(os.path.join(config_root, *rel_path.split("/")))
+
+    def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int]:
+        """
+        Compare committed recipe config (old_recipe_path) to new_recipe_config (config_path).
+
+        - Copies **new** YAML (in new_recipe but not in committed recipe) to changed_files/.
+        - Copies **modified** YAML (same path, different content) to changed_files/.
+        - Respects ignored_files.yml (basename) for both new and modified copies.
+        - Writes deleted_files.csv: paths in committed recipe missing from new_recipe_config.
+        - Writes changed_files_manifest.csv: relative_path + kind (new|modified) for each copy.
+        - Writes new_recipe_not_in_original.csv for paths in new_recipe not in original export.
 
         Returns:
-            Tuple of (changed_count, deleted_count, unchanged_count)
+            Tuple of (added_count, modified_count, deleted_count, unchanged_count,
+                      language_deleted_count, not_in_original_count)
         """
-        # Ensure directories exist
-        if not os.path.exists(self.changed_files_path):
-            os.makedirs(self.changed_files_path)
+        _clear_dir(self.changed_files_path)
+        os.makedirs(self.changed_files_path, exist_ok=True)
 
         # Track statistics
-        changed_count = 0
+        added_count = 0
+        modified_count = 0
         deleted_count = 0
         unchanged_count = 0
         ignored_count = 0
         deleted_files_list = []
+        manifest_rows: List[Tuple[str, str]] = []
 
-        # Get all files in old_recipe_config
-        old_files = set()
-        if os.path.exists(self.old_recipe_path):
-            old_files = {f for f in os.listdir(self.old_recipe_path) if f.endswith('.yml')}
+        # Top-level and language/<lang>/*.yml (matches Drupal export layout)
+        old_files = self._collect_yaml_relpaths(self.old_recipe_path)
+        new_files = self._collect_yaml_relpaths(self.config_path)
 
-        # Get all files in new_recipe_config
-        new_files = set()
-        if os.path.exists(self.config_path):
-            new_files = {f for f in os.listdir(self.config_path) if f.endswith('.yml')}
-
-        # Find deleted files (in old but not in new)
+        # Deleted: in committed recipe but not in new_recipe_config
         deleted_files = old_files - new_files
-        for filename in sorted(deleted_files):
-            deleted_files_list.append(filename)
+        for rel_path in sorted(deleted_files):
+            deleted_files_list.append(rel_path)
             deleted_count += 1
 
-        # Find common files and check for changes
+        # Modified: same path in both, content differs
         common_files = old_files & new_files
-        for filename in sorted(common_files):
-            old_file_path = os.path.join(self.old_recipe_path, filename)
-            new_file_path = os.path.join(self.config_path, filename)
+        for rel_path in sorted(common_files):
+            old_file_path = self._abs_config_path(self.old_recipe_path, rel_path)
+            new_file_path = self._abs_config_path(self.config_path, rel_path)
+            base_name = os.path.basename(rel_path)
 
-            # Compare file contents
             if filecmp.cmp(old_file_path, new_file_path, shallow=False):
-                # Files are identical
                 unchanged_count += 1
             else:
-                # Files have changed - check if ignored
-                if filename in self.ignored_files:
-                    # Skip copying ignored files
+                if base_name in self.ignored_files:
                     ignored_count += 1
                 else:
-                    # Copy to changed_files
-                    changed_dest = os.path.join(self.changed_files_path, filename)
+                    changed_dest = os.path.join(self.changed_files_path, *rel_path.split("/"))
+                    os.makedirs(os.path.dirname(changed_dest), exist_ok=True)
                     shutil.copy2(new_file_path, changed_dest)
-                    changed_count += 1
+                    modified_count += 1
+                    manifest_rows.append((rel_path, "modified"))
+
+        # New: in new_recipe (from original_config via prefixes) but not in committed recipe
+        added_files = new_files - old_files
+        for rel_path in sorted(added_files):
+            new_file_path = self._abs_config_path(self.config_path, rel_path)
+            base_name = os.path.basename(rel_path)
+            if base_name in self.ignored_files:
+                ignored_count += 1
+            else:
+                changed_dest = os.path.join(self.changed_files_path, *rel_path.split("/"))
+                os.makedirs(os.path.dirname(changed_dest), exist_ok=True)
+                shutil.copy2(new_file_path, changed_dest)
+                added_count += 1
+                manifest_rows.append((rel_path, "new"))
+
+        with open(self.changed_files_manifest_csv, "w", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["relative_path", "kind"])
+            for rel_path, kind in sorted(manifest_rows, key=lambda x: (x[1], x[0])):
+                writer.writerow([rel_path, kind])
 
         # Write deleted files to CSV
         with open(self.deleted_files_csv, 'w', newline='') as csvfile:
@@ -211,9 +488,49 @@ class ConfigSelector:
                 writer.writerow([filename])
 
         if ignored_count > 0:
-            print(f"Ignored {ignored_count} changed file(s) as per ignored_files.yml")
+            print(
+                f"Ignored {ignored_count} new/modified file(s) as per ignored_files.yml "
+                "(not copied to changed_files/)."
+            )
+        total_copied = added_count + modified_count
+        if total_copied:
+            print(
+                f"Copied {total_copied} file(s) to changed_files/ "
+                f"({added_count} new vs committed recipe, {modified_count} modified)."
+            )
 
-        return changed_count, deleted_count, unchanged_count
+        language_deleted_count = sum(1 for p in deleted_files_list if p.startswith("language/"))
+        if language_deleted_count:
+            print(
+                f"Deleted list includes {language_deleted_count} language override path(s) "
+                f"(see deleted_files.csv under language/<lang>/)."
+            )
+
+        orig_paths = self._collect_yaml_relpaths(self.original_config_path)
+        new_paths = self._collect_yaml_relpaths(self.config_path)
+        not_in_original = sorted(new_paths - orig_paths)
+        not_in_original_count = len(not_in_original)
+        with open(self.new_recipe_not_in_original_csv, "w", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["filename"])
+            for rel_path in not_in_original:
+                writer.writerow([rel_path])
+        if not_in_original_count:
+            print(
+                f"{not_in_original_count} path(s) in new_recipe_config are not in original_config "
+                f"(see {os.path.basename(self.new_recipe_not_in_original_csv)})."
+            )
+        elif not orig_paths:
+            print("Warning: original_config has no YAML; orphan CSV is empty.")
+
+        return (
+            added_count,
+            modified_count,
+            deleted_count,
+            unchanged_count,
+            language_deleted_count,
+            not_in_original_count,
+        )
 
     def add_prefix(self, prefix: str) -> None:
         """Add a new prefix to the list if it doesn't exist."""
@@ -978,10 +1295,29 @@ def confirm_yaml_cleaning(stdscr, selector):
 
     return confirm_dialog(stdscr, "Copy all matched files to config directory and remove UUID and _core?")
 
-def show_comparison_results(stdscr, changed_count, deleted_count, unchanged_count):
+def show_comparison_results(
+    stdscr,
+    added_count: int,
+    modified_count: int,
+    deleted_count: int,
+    unchanged_count: int,
+    language_deleted_count: int = 0,
+    not_in_original_count: int = 0,
+):
     """Show the results of file comparison."""
     h, w = stdscr.getmaxyx()
-    dialog_h, dialog_w = 10, 60
+    dialog_w = 60
+    total_out = added_count + modified_count
+    # Rows from 3: 5 stats + optional notes + blank + 2 CSV hints; footer at dialog_h - 2
+    lines = (
+        5
+        + (1 if not_in_original_count else 0)
+        + (1 if language_deleted_count else 0)
+        + 1
+        + 2
+    )
+    last_content_row = 2 + lines
+    dialog_h = min(h - 2, max(14, last_content_row + 3))
     dialog_y = (h - dialog_h) // 2
     dialog_x = (w - dialog_w) // 2
 
@@ -989,11 +1325,36 @@ def show_comparison_results(stdscr, changed_count, deleted_count, unchanged_coun
     dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
     dialog_win.box()
     dialog_win.addstr(1, 2, "File Comparison Results", curses.A_BOLD)
-    dialog_win.addstr(3, 4, f"Changed files: {changed_count}")
-    dialog_win.addstr(4, 4, f"Deleted files: {deleted_count}")
-    dialog_win.addstr(5, 4, f"Unchanged files: {unchanged_count}")
-    dialog_win.addstr(7, 4, "Changed files copied to: changed_files/")
-    dialog_win.addstr(8, 4, "Deleted files tracked in: deleted_files.csv")
+    row = 3
+    dialog_win.addstr(row, 4, f"New (not in committed recipe): {added_count}"[: dialog_w - 6])
+    row += 1
+    dialog_win.addstr(row, 4, f"Modified vs committed: {modified_count}"[: dialog_w - 6])
+    row += 1
+    dialog_win.addstr(row, 4, f"Total → changed_files/: {total_out}"[: dialog_w - 6])
+    row += 1
+    deleted_line = f"Deleted (recipe, missing in new): {deleted_count}"
+    if language_deleted_count:
+        deleted_line += f" ({language_deleted_count} language/)"
+    dialog_win.addstr(row, 4, deleted_line[: dialog_w - 6])
+    row += 1
+    dialog_win.addstr(row, 4, f"Unchanged: {unchanged_count}"[: dialog_w - 6])
+    row += 1
+    if not_in_original_count:
+        dialog_win.addstr(
+            row,
+            4,
+            f"Not in original export: {not_in_original_count} (new_recipe_not_in_original.csv)"[
+                : dialog_w - 6
+            ],
+        )
+        row += 1
+    if language_deleted_count:
+        dialog_win.addstr(row, 4, "Language deletions: paths under language/… in deleted CSV")
+        row += 1
+    row += 1
+    dialog_win.addstr(row, 4, "Manifest → changed_files_manifest.csv")
+    row += 1
+    dialog_win.addstr(row, 4, "Deletions → deleted_files.csv")
     dialog_win.addstr(dialog_h - 2, 2, "Press any key to continue...")
     dialog_win.refresh()
 
@@ -1001,36 +1362,26 @@ def show_comparison_results(stdscr, changed_count, deleted_count, unchanged_coun
     stdscr.getch()
 
 def main(stdscr):
-    # Set paths relative to current directory
-    json_path = os.path.join(BASE_DIR, "config_prefixes.json")
-    original_config_path = os.path.join(BASE_DIR, "original_config")
-    config_path = os.path.join(BASE_DIR, "new_recipe_config")
+    # Set paths relative to current directory.
+    jsonPath = os.path.join(BASE_DIR, "config_prefixes.json")
+    configPath = os.path.join(BASE_DIR, "new_recipe_config")
 
-    # Create directories if they don't exist
-    if not os.path.exists(original_config_path):
-        os.makedirs(original_config_path)
-        print(f"Created directory: {original_config_path}")
-        print("Please place your source configuration files in this directory.")
+    # Create directories if they don't exist.
+    os.makedirs(ORIGINAL_CONFIG_DIR, exist_ok=True)
+    os.makedirs(configPath, exist_ok=True)
 
-    if not os.path.exists(config_path):
-        os.makedirs(config_path)
-        print(f"Created directory: {config_path}")
+    # Check if we need to create default JSON file.
+    if not os.path.exists(jsonPath):
+        with open(jsonPath, "w") as f:
+            json.dump({"prefixes": ["put.your.prefixes.here"]}, f, indent=2)
+        print(f"Created default {jsonPath}")
 
-    # Check if we need to create default JSON file
-    if not os.path.exists(json_path):
-        with open(json_path, 'w') as f:
-            json.dump({
-                "prefixes": [
-                    "put.your.prefixes.here",
-                ]
-            }, f, indent=2)
-        print(f"Created default {json_path}")
-
-    # Initialize selector
+    # Initialize selector.
     selector = ConfigSelector(
-        json_path=json_path,
-        original_config_path=original_config_path,
-        config_path=config_path
+        json_path=jsonPath,
+        original_config_path=ORIGINAL_CONFIG_DIR,
+        config_path=configPath,
+        old_recipe_path=RECIPE_CONFIG_DIR,
     )
 
     # Menu state
@@ -1172,8 +1523,23 @@ def main(stdscr):
                 selector.copy_matched_files()
         elif key == ord('m') or key == ord('M'):
             if confirm_dialog(stdscr, "Compare old and new recipe configs and track changes?"):
-                changed_count, deleted_count, unchanged_count = selector.compare_and_track_changes()
-                show_comparison_results(stdscr, changed_count, deleted_count, unchanged_count)
+                (
+                    added_count,
+                    modified_count,
+                    deleted_count,
+                    unchanged_count,
+                    language_deleted_count,
+                    not_in_original_count,
+                ) = selector.compare_and_track_changes()
+                show_comparison_results(
+                    stdscr,
+                    added_count,
+                    modified_count,
+                    deleted_count,
+                    unchanged_count,
+                    language_deleted_count,
+                    not_in_original_count,
+                )
         elif key == ord('s') or key == ord('S'):
             selector.save_prefixes()
         elif key in (10, 13):  # ENTER key to select individual files
@@ -1231,6 +1597,8 @@ def main(stdscr):
                                         selector.matched_files[target_prefix].append(file)
 
 if __name__ == "__main__":
+    forceIngest = "--force" in sys.argv
+    ingest_config(force=forceIngest)
     try:
         curses.wrapper(main)
     except KeyboardInterrupt: