better updates

.gitignore

@@ -14,3 +14,6 @@ recipes/*
 update/*
 .venv
 language_inconsistency.csv
+need_translations.csv
+bin
+AGENT.md

README.md

@@ -181,3 +181,48 @@ Files listed in this YAML will be excluded from the `changed_files/` directory e
 ├── new_recipe_not_in_original.csv              # Paths in new_recipe_config missing from original export
 └── deleted_files.csv                           # Paths in committed recipe missing from new_recipe_config
 ```
+
+## Update WissKI Model
+
+1. **Export config from WissKI** — download a full export from `/admin/config/development/configuration/full/export` and place the archive in `ingest/`.
+
+2. **Run the selector (force)** — activate the virtualenv and start the script:
+
+   ```bash
+   source .venv/bin/activate
+   python3 config_selector.py --force
+   ```
+
+3. **Copy and compare** — choose `copy`, then `compare`.
+
+4. **Review changes** — inspect `changed_files_manifest.csv` and `deleted_files.csv`; control scrap and failures before continuing.
+
+5. **Update the recipe** — run the selector again (without `--force`) and choose `update`:
+
+   ```bash
+   python3 config_selector.py
+   ```
+
+6. **Commit and push** — stage changes, then push with the helper script:
+
+   ```bash
+   git add /opt/drupal/private-files/drupal_config_extract/update
+   bin/wisski-push.sh <your-name> "your commit message"
+   ```
+
+7. **Pull the recipe** — update the local recipe checkout:
+
+   ```bash
+   cd /opt/drupal/recipes/wisski_default_data_model
+   git pull
+   ```
+
+8. **Reset the WissKI instance** — delete bundle, fields, and pathbuilder on the instance.
+
+9. **Re-apply the recipe** — from the Drupal docroot:
+
+   ```bash
+   drush cr
+   drush recipe ../recipes/wisski_default_data_model
+   drush wisski-core:recreate-menus
+   ```

config_selector.py

@@ -109,7 +109,7 @@ def ingest_config(force: bool = False) -> None:
 class ConfigSelector:
     def __init__(self, json_path: str, original_config_path: str, config_path: str,
                  old_recipe_path: str = None, changed_files_path: str = None, deleted_files_csv: str = None,
-                 ignored_files_yml: str = None):
+                 ignored_files_yml: str = None, translatable_prefixes_json: str = None):
         self.json_path = json_path
         self.original_config_path = original_config_path
         self.config_path = config_path
@@ -118,13 +118,19 @@ class ConfigSelector:
         self.deleted_files_csv = deleted_files_csv or os.path.join(BASE_DIR, "deleted_files.csv")
         self.new_recipe_not_in_original_csv = os.path.join(BASE_DIR, "new_recipe_not_in_original.csv")
         self.language_inconsistency_csv = os.path.join(BASE_DIR, "language_inconsistency.csv")
+        self.need_translations_csv = os.path.join(BASE_DIR, "need_translations.csv")
         self.changed_files_manifest_csv = os.path.join(BASE_DIR, "changed_files_manifest.csv")
         self.ignored_files_yml = ignored_files_yml or os.path.join(BASE_DIR, "ignored_files.yml")
+        self.translatable_prefixes_json = (
+            translatable_prefixes_json or os.path.join(BASE_DIR, "translatable_config_prefixes.json")
+        )
         self.prefixes: List[str] = []
+        self.translatable_prefixes: List[str] = []
         self.matched_files: Dict[str, List[str]] = {}
         self.unmatched_files: Dict[str, List[str]] = {}
         self.ignored_files: Set[str] = set()
         self.load_prefixes()
+        self.load_translatable_prefixes()
         self.load_ignored_files()
         self.scan_configs()
 
@@ -144,6 +150,26 @@ class ConfigSelector:
             print(f"Error loading JSON file: {e}")
             exit(1)
 
+    def load_translatable_prefixes(self) -> None:
+        """Load config prefixes that require German (de) translations from JSON."""
+        self.translatable_prefixes = []
+        if not os.path.exists(self.translatable_prefixes_json):
+            return
+        try:
+            with open(self.translatable_prefixes_json, "r") as f:
+                data = json.load(f)
+                if isinstance(data, list):
+                    self.translatable_prefixes = data
+                elif isinstance(data, dict) and "prefixes" in data:
+                    self.translatable_prefixes = data["prefixes"]
+                else:
+                    print(
+                        "Warning: translatable_config_prefixes.json should contain a list "
+                        "or a dict with 'prefixes' key"
+                    )
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"Warning: error loading translatable prefixes JSON: {e}")
+
     def load_ignored_files(self) -> None:
         """Load ignored file names from YAML file."""
         self.ignored_files = set()
@@ -260,7 +286,7 @@ class ConfigSelector:
             overridesCopied += 1
         return overridesCopied
 
-    def copy_matched_files(self) -> None:
+    def copy_matched_files(self, quiet: bool = False) -> None:
         """Copy all matched files from original_config to config, removing UUID and _core structures."""
         if not os.path.exists(self.config_path):
             os.makedirs(self.config_path)
@@ -296,6 +322,7 @@ class ConfigSelector:
                 langCopied += 1
                 overrideCopied += self._copy_with_language_overrides(langFilename)
 
+        if not quiet:
             print(
                 f"Copied {copiedCount} files to {self.config_path} "
                 f"({processedCount} processed to remove UUIDs and _core structures)"
@@ -305,13 +332,13 @@ class ConfigSelector:
             if overrideCopied:
                 print(f"Also copied {overrideCopied} language override files from original_config/language/.")
         pruned_export = self._prune_new_recipe_not_in_original()
-        if pruned_export:
+        if pruned_export and not quiet:
             print(
                 f"Removed {pruned_export} file(s) from new_recipe_config that are not in original_config "
                 f"(stale vs current export)."
             )
         pruned = self._prune_stale_language_overrides()
-        if pruned:
+        if pruned and not quiet:
             print(f"Removed {pruned} stale language override file(s) (no longer in export or missing base config).")
 
     @staticmethod
@@ -447,6 +474,53 @@ class ConfigSelector:
             for rel_path in inconsistent:
                 writer.writerow([rel_path])
 
+    @staticmethod
+    def _matches_any_prefix(filename: str, prefixes: List[str]) -> bool:
+        return any(filename.startswith(prefix) for prefix in prefixes)
+
+    def _collect_missing_de_translations(self) -> List[str]:
+        """Base configs in config_path matching translatable prefixes without language/de/.
+
+        A translation is considered present when language/de/{filename} exists in
+        new_recipe_config or in the committed recipe (old_recipe_path).
+        """
+        missing: List[str] = []
+        if not self.translatable_prefixes:
+            return missing
+
+        def de_filenames(config_root: str) -> Set[str]:
+            de_dir = os.path.join(config_root, "language", "de")
+            if not os.path.isdir(de_dir):
+                return set()
+            try:
+                return {
+                    name
+                    for name in os.listdir(de_dir)
+                    if name.endswith(".yml")
+                }
+            except OSError:
+                return set()
+
+        try:
+            de_files = de_filenames(self.config_path) | de_filenames(self.old_recipe_path)
+            for name in os.listdir(self.config_path):
+                if not name.endswith(".yml"):
+                    continue
+                if not self._matches_any_prefix(name, self.translatable_prefixes):
+                    continue
+                if name not in de_files:
+                    missing.append(name)
+        except OSError:
+            pass
+        return sorted(missing)
+
+    def _write_need_translations_csv(self, missing: List[str]) -> None:
+        with open(self.need_translations_csv, "w", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["filename"])
+            for filename in missing:
+                writer.writerow([filename])
+
     def delete_language_inconsistencies(self) -> int:
         """Remove language/*/*.yml in config_path that have no base config pendant."""
         inconsistent = self._collect_language_inconsistencies()
@@ -477,7 +551,7 @@ class ConfigSelector:
                     paths.append(name)
         return paths
 
-    def update_recipe(self) -> Tuple[int, int, int, str]:
+    def update_recipe(self, quiet: bool = False) -> Tuple[int, int, int, str]:
         """Copy committed recipe to update/<recipe_name>/, apply deletions and changed_files/.
 
         Returns:
@@ -512,6 +586,7 @@ class ConfigSelector:
             shutil.copy2(src, dst)
             files_applied += 1
 
+        if not quiet:
             print(
                 f"Updated recipe written to {update_dir}/ "
                 f"({files_deleted} deleted from config, {files_applied} copied from changed_files/, "
@@ -519,7 +594,7 @@ class ConfigSelector:
             )
         return files_deleted, files_applied, deletes_missing, update_dir
 
-    def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int, int]:
+    def compare_and_track_changes(self, quiet: bool = False) -> Tuple[int, int, int, int, int, int, int, int]:
         """
         Compare committed recipe config (old_recipe_path) to new_recipe_config (config_path).
 
@@ -530,11 +605,12 @@ class ConfigSelector:
         - Writes changed_files_manifest.csv: relative_path + kind (new|modified) for each copy.
         - Writes new_recipe_not_in_original.csv for paths in new_recipe not in original export.
         - Writes language_inconsistency.csv for language/*/*.yml without a base config pendant.
+        - Writes need_translations.csv for translatable-prefix configs missing language/de/.
 
         Returns:
             Tuple of (added_count, modified_count, deleted_count, unchanged_count,
                       language_deleted_count, not_in_original_count,
-                      language_inconsistency_count)
+                      language_inconsistency_count, need_translations_count)
         """
         _clear_dir(self.changed_files_path)
         os.makedirs(self.changed_files_path, exist_ok=True)
@@ -604,20 +680,20 @@ class ConfigSelector:
             for filename in deleted_files_list:
                 writer.writerow([filename])
 
-        if ignored_count > 0:
+        if ignored_count > 0 and not quiet:
             print(
                 f"Ignored {ignored_count} new/modified file(s) as per ignored_files.yml "
                 "(not copied to changed_files/)."
             )
         total_copied = added_count + modified_count
-        if total_copied:
+        if total_copied and not quiet:
             print(
                 f"Copied {total_copied} file(s) to changed_files/ "
                 f"({added_count} new vs committed recipe, {modified_count} modified)."
             )
 
         language_deleted_count = sum(1 for p in deleted_files_list if p.startswith("language/"))
-        if language_deleted_count:
+        if language_deleted_count and not quiet:
             print(
                 f"Deleted list includes {language_deleted_count} language override path(s) "
                 f"(see deleted_files.csv under language/<lang>/)."
@@ -632,24 +708,34 @@ class ConfigSelector:
             writer.writerow(["filename"])
             for rel_path in not_in_original:
                 writer.writerow([rel_path])
-        if not_in_original_count:
+        if not_in_original_count and not quiet:
             print(
                 f"{not_in_original_count} path(s) in new_recipe_config are not in original_config "
                 f"(see {os.path.basename(self.new_recipe_not_in_original_csv)})."
             )
-        elif not orig_paths:
+        elif not orig_paths and not quiet:
             print("Warning: original_config has no YAML; orphan CSV is empty.")
 
         language_inconsistent = self._collect_language_inconsistencies()
         language_inconsistency_count = len(language_inconsistent)
         self._write_language_inconsistency_csv(language_inconsistent)
-        if language_inconsistency_count:
+        if language_inconsistency_count and not quiet:
             print(
                 f"{language_inconsistency_count} language override(s) in new_recipe_config "
                 f"have no base config pendant "
                 f"(see {os.path.basename(self.language_inconsistency_csv)})."
             )
 
+        missing_de_translations = self._collect_missing_de_translations()
+        need_translations_count = len(missing_de_translations)
+        self._write_need_translations_csv(missing_de_translations)
+        if need_translations_count and not quiet:
+            print(
+                f"{need_translations_count} translatable config(s) in new_recipe_config "
+                f"have no German (de) translation "
+                f"(see {os.path.basename(self.need_translations_csv)})."
+            )
+
         return (
             added_count,
             modified_count,
@@ -658,6 +744,7 @@ class ConfigSelector:
             language_deleted_count,
             not_in_original_count,
             language_inconsistency_count,
+            need_translations_count,
         )
 
     def add_prefix(self, prefix: str) -> None:
@@ -1442,6 +1529,42 @@ def _curses_addstr_clipped(win, y: int, x: int, text: str, max_width: int, attrs
     except curses.error:
         pass
 
+def _show_text_dialog(stdscr, title: str, content_lines: List[str]) -> None:
+    """Show a bordered dialog; clears the screen first to avoid TUI bleed-through."""
+    stdscr.clear()
+    stdscr.refresh()
+    h, w = stdscr.getmaxyx()
+    if h < 8 or w < 24:
+        return
+
+    dialog_w = min(72, w - 4)
+    inner_h = h - 6
+    lines = list(content_lines)
+    if len(lines) > inner_h:
+        hidden = len(lines) - inner_h + 1
+        lines = lines[: inner_h - 1]
+        lines.append(f"... +{hidden} more (see CSV files)")
+
+    dialog_h = min(h - 2, max(8, len(lines) + 4))
+    dialog_y = max(0, (h - dialog_h) // 2)
+    dialog_x = max(0, (w - dialog_w) // 2)
+    if dialog_h <= 0 or dialog_w <= 0:
+        return
+
+    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
+    dialog_win.box()
+    _curses_addstr_clipped(dialog_win, 1, 2, title, dialog_w, curses.A_BOLD)
+    for i, line in enumerate(lines):
+        row = 2 + i
+        if row >= dialog_h - 2:
+            break
+        _curses_addstr_clipped(dialog_win, row, 2, line, dialog_w)
+    _curses_addstr_clipped(
+        dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w
+    )
+    dialog_win.refresh()
+    stdscr.getch()
+
 def show_info_dialog(stdscr, message: str) -> None:
     """Show a simple message dialog."""
     h, w = stdscr.getmaxyx()
@@ -1468,8 +1591,6 @@ def show_update_recipe_results(
     update_dir: str,
 ) -> None:
     """Show the results of building update/<recipe_name>/ from the committed recipe."""
-    h, w = stdscr.getmaxyx()
-    dialog_w = min(60, w - 4)
     content_lines = [
         f"Recipe copy: {update_dir}/",
         f"Deleted from config: {files_deleted}",
@@ -1477,26 +1598,7 @@ def show_update_recipe_results(
     ]
     if deletes_missing:
         content_lines.append(f"Delete CSV entries not in copy: {deletes_missing}")
-    content_start = 3
-    dialog_h = min(h - 2, max(12, content_start + len(content_lines) + 3))
-    dialog_y = max(0, (h - dialog_h) // 2)
-    dialog_x = max(0, (w - dialog_w) // 2)
-    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
-    dialog_win.box()
-    _curses_addstr_clipped(
-        dialog_win, 1, 2, "Recipe Update Results", dialog_w, curses.A_BOLD
-    )
-    max_footer_row = dialog_h - 3
-    for i, line in enumerate(content_lines):
-        row = content_start + i
-        if row > max_footer_row:
-            break
-        _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w)
-    _curses_addstr_clipped(
-        dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w
-    )
-    dialog_win.refresh()
-    stdscr.getch()
+    _show_text_dialog(stdscr, "Recipe Update Results", content_lines)
 
 def show_comparison_results(
     stdscr,
@@ -1507,10 +1609,9 @@ def show_comparison_results(
     language_deleted_count: int = 0,
     not_in_original_count: int = 0,
     language_inconsistency_count: int = 0,
+    need_translations_count: int = 0,
 ):
     """Show the results of file comparison."""
-    h, w = stdscr.getmaxyx()
-    dialog_w = min(60, w - 4)
     total_out = added_count + modified_count
     deleted_line = f"Deleted (recipe, missing in new): {deleted_count}"
     if language_deleted_count:
@@ -1529,14 +1630,17 @@ def show_comparison_results(
             "(new_recipe_not_in_original.csv)"
         )
     if language_deleted_count:
-        content_lines.append(
-            "Language deletions: paths under language/ in deleted CSV"
-        )
+        content_lines.append("Language deletions: see deleted_files.csv")
     if language_inconsistency_count:
         content_lines.append(
             f"Language inconsistency: {language_inconsistency_count} "
             "(language_inconsistency.csv)"
         )
+    if need_translations_count:
+        content_lines.append(
+            f"Missing de translation: {need_translations_count} "
+            "(need_translations.csv)"
+        )
     content_lines.extend(
         [
             "",
@@ -1544,31 +1648,7 @@ def show_comparison_results(
             "Deletions -> deleted_files.csv",
         ]
     )
-
-    content_start = 3
-    # Leave one blank row between content and footer (footer at dialog_h - 2).
-    dialog_h = min(h - 2, max(14, content_start + len(content_lines) + 3))
-    dialog_y = max(0, (h - dialog_h) // 2)
-    dialog_x = max(0, (w - dialog_w) // 2)
-
-    dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x)
-    dialog_win.box()
-    _curses_addstr_clipped(
-        dialog_win, 1, 2, "File Comparison Results", dialog_w, curses.A_BOLD
-    )
-    max_footer_row = dialog_h - 3
-    for i, line in enumerate(content_lines):
-        row = content_start + i
-        if row > max_footer_row:
-            break
-        _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w)
-    _curses_addstr_clipped(
-        dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w
-    )
-    dialog_win.refresh()
-
-    # Wait for key
-    stdscr.getch()
+    _show_text_dialog(stdscr, "File Comparison Results", content_lines)
 
 def main(stdscr):
     # Set paths relative to current directory.
@@ -1585,6 +1665,25 @@ def main(stdscr):
             json.dump({"prefixes": ["put.your.prefixes.here"]}, f, indent=2)
         print(f"Created default {jsonPath}")
 
+    translatablePrefixesPath = os.path.join(BASE_DIR, "translatable_config_prefixes.json")
+    if not os.path.exists(translatablePrefixesPath):
+        with open(translatablePrefixesPath, "w") as f:
+            json.dump(
+                {
+                    "prefixes": [
+                        "core.entity_form_display",
+                        "core.entity_form_mode",
+                        "field.field.wisski_individual",
+                        "field.storage.wisski_individual",
+                        "views.view",
+                        "wisski_core.wisski_bundle",
+                    ]
+                },
+                f,
+                indent=2,
+            )
+        print(f"Created default {translatablePrefixesPath}")
+
     # Initialize selector.
     selector = ConfigSelector(
         json_path=jsonPath,
@@ -1729,7 +1828,7 @@ def main(stdscr):
                     selected_idx = 0
         elif key == ord('c') or key == ord('C'):
             if confirm_yaml_cleaning(stdscr, selector):
-                selector.copy_matched_files()
+                selector.copy_matched_files(quiet=True)
         elif key == ord('m') or key == ord('M'):
             if confirm_dialog(stdscr, "Compare old and new recipe configs and track changes?"):
                 (
@@ -1740,7 +1839,8 @@ def main(stdscr):
                     language_deleted_count,
                     not_in_original_count,
                     language_inconsistency_count,
-                ) = selector.compare_and_track_changes()
+                    need_translations_count,
+                ) = selector.compare_and_track_changes(quiet=True)
                 show_comparison_results(
                     stdscr,
                     added_count,
@@ -1750,6 +1850,7 @@ def main(stdscr):
                     language_deleted_count,
                     not_in_original_count,
                     language_inconsistency_count,
+                    need_translations_count,
                 )
         elif key == ord('i') or key == ord('I'):
             inconsistent = selector._collect_language_inconsistencies()
@@ -1772,7 +1873,7 @@ def main(stdscr):
             ):
                 try:
                     files_deleted, files_applied, deletes_missing, update_dir = (
-                        selector.update_recipe()
+                        selector.update_recipe(quiet=True)
                     )
                     show_update_recipe_results(
                         stdscr,

ignored_files.yml

@@ -4,6 +4,15 @@ field:
 field:
   name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__ownership
   reason: because we need target id instead of uuid
+field:
+  name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__custody
+  reason: because we need target id instead of uuid
+field:
+  name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__stay
+  reason: because we need target id instead of uuid
 views:
   name: views.view.block_content
   reason: false positive
+language:
+  name: language.entity.de
+  reason: because the inital page setup was german

translatable_config_prefixes.json

@@ -0,0 +1,10 @@
+{
+  "prefixes": [
+    "core.entity_form_display",
+    "core.entity_form_mode",
+    "field.field.wisski_individual",
+    "field.storage.wisski_individual",
+    "views.view",
+    "wisski_core.wisski_bundle"
+  ]
+}