diff --git a/.gitignore b/.gitignore index cdcc649..fbb740b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ ignored_files.yml recipes/* update/* .venv -language_inconsistency.csv \ No newline at end of file +language_inconsistency.csv +need_translations.csv +bin +AGENT.md \ No newline at end of file diff --git a/README.md b/README.md index 6b7a38a..0555d4a 100644 --- a/README.md +++ b/README.md @@ -181,3 +181,48 @@ Files listed in this YAML will be excluded from the `changed_files/` directory e ├── new_recipe_not_in_original.csv # Paths in new_recipe_config missing from original export └── deleted_files.csv # Paths in committed recipe missing from new_recipe_config ``` + +## Update WissKI Model + +1. **Export config from WissKI** — download a full export from `/admin/config/development/configuration/full/export` and place the archive in `ingest/`. + +2. **Run the selector (force)** — activate the virtualenv and start the script: + + ```bash + source .venv/bin/activate + python3 config_selector.py --force + ``` + +3. **Copy and compare** — choose `copy`, then `compare`. + +4. **Review changes** — inspect `changed_files_manifest.csv` and `deleted_files.csv`; control scrap and failures before continuing. + +5. **Update the recipe** — run the selector again (without `--force`) and choose `update`: + + ```bash + python3 config_selector.py + ``` + +6. **Commit and push** — stage changes, then push with the helper script: + + ```bash + git add /opt/drupal/private-files/drupal_config_extract/update + bin/wisski-push.sh "your commit message" + ``` + +7. **Pull the recipe** — update the local recipe checkout: + + ```bash + cd /opt/drupal/recipes/wisski_default_data_model + git pull + ``` + +8. **Reset the WissKI instance** — delete bundle, fields, and pathbuilder on the instance. + +9. **Re-apply the recipe** — from the Drupal docroot: + + ```bash + drush cr + drush recipe ../recipes/wisski_default_data_model + drush wisski-core:recreate-menus + ``` \ No newline at end of file diff --git a/config_selector.py b/config_selector.py index 92ce2b0..7eddec5 100644 --- a/config_selector.py +++ b/config_selector.py @@ -109,7 +109,7 @@ def ingest_config(force: bool = False) -> None: class ConfigSelector: def __init__(self, json_path: str, original_config_path: str, config_path: str, old_recipe_path: str = None, changed_files_path: str = None, deleted_files_csv: str = None, - ignored_files_yml: str = None): + ignored_files_yml: str = None, translatable_prefixes_json: str = None): self.json_path = json_path self.original_config_path = original_config_path self.config_path = config_path @@ -118,13 +118,19 @@ class ConfigSelector: self.deleted_files_csv = deleted_files_csv or os.path.join(BASE_DIR, "deleted_files.csv") self.new_recipe_not_in_original_csv = os.path.join(BASE_DIR, "new_recipe_not_in_original.csv") self.language_inconsistency_csv = os.path.join(BASE_DIR, "language_inconsistency.csv") + self.need_translations_csv = os.path.join(BASE_DIR, "need_translations.csv") self.changed_files_manifest_csv = os.path.join(BASE_DIR, "changed_files_manifest.csv") self.ignored_files_yml = ignored_files_yml or os.path.join(BASE_DIR, "ignored_files.yml") + self.translatable_prefixes_json = ( + translatable_prefixes_json or os.path.join(BASE_DIR, "translatable_config_prefixes.json") + ) self.prefixes: List[str] = [] + self.translatable_prefixes: List[str] = [] self.matched_files: Dict[str, List[str]] = {} self.unmatched_files: Dict[str, List[str]] = {} self.ignored_files: Set[str] = set() self.load_prefixes() + self.load_translatable_prefixes() self.load_ignored_files() self.scan_configs() @@ -144,6 +150,26 @@ class ConfigSelector: print(f"Error loading JSON file: {e}") exit(1) + def load_translatable_prefixes(self) -> None: + """Load config prefixes that require German (de) translations from JSON.""" + self.translatable_prefixes = [] + if not os.path.exists(self.translatable_prefixes_json): + return + try: + with open(self.translatable_prefixes_json, "r") as f: + data = json.load(f) + if isinstance(data, list): + self.translatable_prefixes = data + elif isinstance(data, dict) and "prefixes" in data: + self.translatable_prefixes = data["prefixes"] + else: + print( + "Warning: translatable_config_prefixes.json should contain a list " + "or a dict with 'prefixes' key" + ) + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: error loading translatable prefixes JSON: {e}") + def load_ignored_files(self) -> None: """Load ignored file names from YAML file.""" self.ignored_files = set() @@ -260,7 +286,7 @@ class ConfigSelector: overridesCopied += 1 return overridesCopied - def copy_matched_files(self) -> None: + def copy_matched_files(self, quiet: bool = False) -> None: """Copy all matched files from original_config to config, removing UUID and _core structures.""" if not os.path.exists(self.config_path): os.makedirs(self.config_path) @@ -296,22 +322,23 @@ class ConfigSelector: langCopied += 1 overrideCopied += self._copy_with_language_overrides(langFilename) - print( - f"Copied {copiedCount} files to {self.config_path} " - f"({processedCount} processed to remove UUIDs and _core structures)" - ) - if langCopied: - print(f"Also copied {langCopied} corresponding language.content_settings files.") - if overrideCopied: - print(f"Also copied {overrideCopied} language override files from original_config/language/.") + if not quiet: + print( + f"Copied {copiedCount} files to {self.config_path} " + f"({processedCount} processed to remove UUIDs and _core structures)" + ) + if langCopied: + print(f"Also copied {langCopied} corresponding language.content_settings files.") + if overrideCopied: + print(f"Also copied {overrideCopied} language override files from original_config/language/.") pruned_export = self._prune_new_recipe_not_in_original() - if pruned_export: + if pruned_export and not quiet: print( f"Removed {pruned_export} file(s) from new_recipe_config that are not in original_config " f"(stale vs current export)." ) pruned = self._prune_stale_language_overrides() - if pruned: + if pruned and not quiet: print(f"Removed {pruned} stale language override file(s) (no longer in export or missing base config).") @staticmethod @@ -447,6 +474,53 @@ class ConfigSelector: for rel_path in inconsistent: writer.writerow([rel_path]) + @staticmethod + def _matches_any_prefix(filename: str, prefixes: List[str]) -> bool: + return any(filename.startswith(prefix) for prefix in prefixes) + + def _collect_missing_de_translations(self) -> List[str]: + """Base configs in config_path matching translatable prefixes without language/de/. + + A translation is considered present when language/de/{filename} exists in + new_recipe_config or in the committed recipe (old_recipe_path). + """ + missing: List[str] = [] + if not self.translatable_prefixes: + return missing + + def de_filenames(config_root: str) -> Set[str]: + de_dir = os.path.join(config_root, "language", "de") + if not os.path.isdir(de_dir): + return set() + try: + return { + name + for name in os.listdir(de_dir) + if name.endswith(".yml") + } + except OSError: + return set() + + try: + de_files = de_filenames(self.config_path) | de_filenames(self.old_recipe_path) + for name in os.listdir(self.config_path): + if not name.endswith(".yml"): + continue + if not self._matches_any_prefix(name, self.translatable_prefixes): + continue + if name not in de_files: + missing.append(name) + except OSError: + pass + return sorted(missing) + + def _write_need_translations_csv(self, missing: List[str]) -> None: + with open(self.need_translations_csv, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["filename"]) + for filename in missing: + writer.writerow([filename]) + def delete_language_inconsistencies(self) -> int: """Remove language/*/*.yml in config_path that have no base config pendant.""" inconsistent = self._collect_language_inconsistencies() @@ -477,7 +551,7 @@ class ConfigSelector: paths.append(name) return paths - def update_recipe(self) -> Tuple[int, int, int, str]: + def update_recipe(self, quiet: bool = False) -> Tuple[int, int, int, str]: """Copy committed recipe to update//, apply deletions and changed_files/. Returns: @@ -512,14 +586,15 @@ class ConfigSelector: shutil.copy2(src, dst) files_applied += 1 - print( - f"Updated recipe written to {update_dir}/ " - f"({files_deleted} deleted from config, {files_applied} copied from changed_files/, " - f"{deletes_missing} delete entries not present in copy)." - ) + if not quiet: + print( + f"Updated recipe written to {update_dir}/ " + f"({files_deleted} deleted from config, {files_applied} copied from changed_files/, " + f"{deletes_missing} delete entries not present in copy)." + ) return files_deleted, files_applied, deletes_missing, update_dir - def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int, int]: + def compare_and_track_changes(self, quiet: bool = False) -> Tuple[int, int, int, int, int, int, int, int]: """ Compare committed recipe config (old_recipe_path) to new_recipe_config (config_path). @@ -530,11 +605,12 @@ class ConfigSelector: - Writes changed_files_manifest.csv: relative_path + kind (new|modified) for each copy. - Writes new_recipe_not_in_original.csv for paths in new_recipe not in original export. - Writes language_inconsistency.csv for language/*/*.yml without a base config pendant. + - Writes need_translations.csv for translatable-prefix configs missing language/de/. Returns: Tuple of (added_count, modified_count, deleted_count, unchanged_count, language_deleted_count, not_in_original_count, - language_inconsistency_count) + language_inconsistency_count, need_translations_count) """ _clear_dir(self.changed_files_path) os.makedirs(self.changed_files_path, exist_ok=True) @@ -604,20 +680,20 @@ class ConfigSelector: for filename in deleted_files_list: writer.writerow([filename]) - if ignored_count > 0: + if ignored_count > 0 and not quiet: print( f"Ignored {ignored_count} new/modified file(s) as per ignored_files.yml " "(not copied to changed_files/)." ) total_copied = added_count + modified_count - if total_copied: + if total_copied and not quiet: print( f"Copied {total_copied} file(s) to changed_files/ " f"({added_count} new vs committed recipe, {modified_count} modified)." ) language_deleted_count = sum(1 for p in deleted_files_list if p.startswith("language/")) - if language_deleted_count: + if language_deleted_count and not quiet: print( f"Deleted list includes {language_deleted_count} language override path(s) " f"(see deleted_files.csv under language//)." @@ -632,24 +708,34 @@ class ConfigSelector: writer.writerow(["filename"]) for rel_path in not_in_original: writer.writerow([rel_path]) - if not_in_original_count: + if not_in_original_count and not quiet: print( f"{not_in_original_count} path(s) in new_recipe_config are not in original_config " f"(see {os.path.basename(self.new_recipe_not_in_original_csv)})." ) - elif not orig_paths: + elif not orig_paths and not quiet: print("Warning: original_config has no YAML; orphan CSV is empty.") language_inconsistent = self._collect_language_inconsistencies() language_inconsistency_count = len(language_inconsistent) self._write_language_inconsistency_csv(language_inconsistent) - if language_inconsistency_count: + if language_inconsistency_count and not quiet: print( f"{language_inconsistency_count} language override(s) in new_recipe_config " f"have no base config pendant " f"(see {os.path.basename(self.language_inconsistency_csv)})." ) + missing_de_translations = self._collect_missing_de_translations() + need_translations_count = len(missing_de_translations) + self._write_need_translations_csv(missing_de_translations) + if need_translations_count and not quiet: + print( + f"{need_translations_count} translatable config(s) in new_recipe_config " + f"have no German (de) translation " + f"(see {os.path.basename(self.need_translations_csv)})." + ) + return ( added_count, modified_count, @@ -658,6 +744,7 @@ class ConfigSelector: language_deleted_count, not_in_original_count, language_inconsistency_count, + need_translations_count, ) def add_prefix(self, prefix: str) -> None: @@ -1442,6 +1529,42 @@ def _curses_addstr_clipped(win, y: int, x: int, text: str, max_width: int, attrs except curses.error: pass +def _show_text_dialog(stdscr, title: str, content_lines: List[str]) -> None: + """Show a bordered dialog; clears the screen first to avoid TUI bleed-through.""" + stdscr.clear() + stdscr.refresh() + h, w = stdscr.getmaxyx() + if h < 8 or w < 24: + return + + dialog_w = min(72, w - 4) + inner_h = h - 6 + lines = list(content_lines) + if len(lines) > inner_h: + hidden = len(lines) - inner_h + 1 + lines = lines[: inner_h - 1] + lines.append(f"... +{hidden} more (see CSV files)") + + dialog_h = min(h - 2, max(8, len(lines) + 4)) + dialog_y = max(0, (h - dialog_h) // 2) + dialog_x = max(0, (w - dialog_w) // 2) + if dialog_h <= 0 or dialog_w <= 0: + return + + dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) + dialog_win.box() + _curses_addstr_clipped(dialog_win, 1, 2, title, dialog_w, curses.A_BOLD) + for i, line in enumerate(lines): + row = 2 + i + if row >= dialog_h - 2: + break + _curses_addstr_clipped(dialog_win, row, 2, line, dialog_w) + _curses_addstr_clipped( + dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w + ) + dialog_win.refresh() + stdscr.getch() + def show_info_dialog(stdscr, message: str) -> None: """Show a simple message dialog.""" h, w = stdscr.getmaxyx() @@ -1468,8 +1591,6 @@ def show_update_recipe_results( update_dir: str, ) -> None: """Show the results of building update// from the committed recipe.""" - h, w = stdscr.getmaxyx() - dialog_w = min(60, w - 4) content_lines = [ f"Recipe copy: {update_dir}/", f"Deleted from config: {files_deleted}", @@ -1477,26 +1598,7 @@ def show_update_recipe_results( ] if deletes_missing: content_lines.append(f"Delete CSV entries not in copy: {deletes_missing}") - content_start = 3 - dialog_h = min(h - 2, max(12, content_start + len(content_lines) + 3)) - dialog_y = max(0, (h - dialog_h) // 2) - dialog_x = max(0, (w - dialog_w) // 2) - dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) - dialog_win.box() - _curses_addstr_clipped( - dialog_win, 1, 2, "Recipe Update Results", dialog_w, curses.A_BOLD - ) - max_footer_row = dialog_h - 3 - for i, line in enumerate(content_lines): - row = content_start + i - if row > max_footer_row: - break - _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w) - _curses_addstr_clipped( - dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w - ) - dialog_win.refresh() - stdscr.getch() + _show_text_dialog(stdscr, "Recipe Update Results", content_lines) def show_comparison_results( stdscr, @@ -1507,10 +1609,9 @@ def show_comparison_results( language_deleted_count: int = 0, not_in_original_count: int = 0, language_inconsistency_count: int = 0, + need_translations_count: int = 0, ): """Show the results of file comparison.""" - h, w = stdscr.getmaxyx() - dialog_w = min(60, w - 4) total_out = added_count + modified_count deleted_line = f"Deleted (recipe, missing in new): {deleted_count}" if language_deleted_count: @@ -1529,14 +1630,17 @@ def show_comparison_results( "(new_recipe_not_in_original.csv)" ) if language_deleted_count: - content_lines.append( - "Language deletions: paths under language/ in deleted CSV" - ) + content_lines.append("Language deletions: see deleted_files.csv") if language_inconsistency_count: content_lines.append( f"Language inconsistency: {language_inconsistency_count} " "(language_inconsistency.csv)" ) + if need_translations_count: + content_lines.append( + f"Missing de translation: {need_translations_count} " + "(need_translations.csv)" + ) content_lines.extend( [ "", @@ -1544,31 +1648,7 @@ def show_comparison_results( "Deletions -> deleted_files.csv", ] ) - - content_start = 3 - # Leave one blank row between content and footer (footer at dialog_h - 2). - dialog_h = min(h - 2, max(14, content_start + len(content_lines) + 3)) - dialog_y = max(0, (h - dialog_h) // 2) - dialog_x = max(0, (w - dialog_w) // 2) - - dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) - dialog_win.box() - _curses_addstr_clipped( - dialog_win, 1, 2, "File Comparison Results", dialog_w, curses.A_BOLD - ) - max_footer_row = dialog_h - 3 - for i, line in enumerate(content_lines): - row = content_start + i - if row > max_footer_row: - break - _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w) - _curses_addstr_clipped( - dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w - ) - dialog_win.refresh() - - # Wait for key - stdscr.getch() + _show_text_dialog(stdscr, "File Comparison Results", content_lines) def main(stdscr): # Set paths relative to current directory. @@ -1585,6 +1665,25 @@ def main(stdscr): json.dump({"prefixes": ["put.your.prefixes.here"]}, f, indent=2) print(f"Created default {jsonPath}") + translatablePrefixesPath = os.path.join(BASE_DIR, "translatable_config_prefixes.json") + if not os.path.exists(translatablePrefixesPath): + with open(translatablePrefixesPath, "w") as f: + json.dump( + { + "prefixes": [ + "core.entity_form_display", + "core.entity_form_mode", + "field.field.wisski_individual", + "field.storage.wisski_individual", + "views.view", + "wisski_core.wisski_bundle", + ] + }, + f, + indent=2, + ) + print(f"Created default {translatablePrefixesPath}") + # Initialize selector. selector = ConfigSelector( json_path=jsonPath, @@ -1729,7 +1828,7 @@ def main(stdscr): selected_idx = 0 elif key == ord('c') or key == ord('C'): if confirm_yaml_cleaning(stdscr, selector): - selector.copy_matched_files() + selector.copy_matched_files(quiet=True) elif key == ord('m') or key == ord('M'): if confirm_dialog(stdscr, "Compare old and new recipe configs and track changes?"): ( @@ -1740,7 +1839,8 @@ def main(stdscr): language_deleted_count, not_in_original_count, language_inconsistency_count, - ) = selector.compare_and_track_changes() + need_translations_count, + ) = selector.compare_and_track_changes(quiet=True) show_comparison_results( stdscr, added_count, @@ -1750,6 +1850,7 @@ def main(stdscr): language_deleted_count, not_in_original_count, language_inconsistency_count, + need_translations_count, ) elif key == ord('i') or key == ord('I'): inconsistent = selector._collect_language_inconsistencies() @@ -1772,7 +1873,7 @@ def main(stdscr): ): try: files_deleted, files_applied, deletes_missing, update_dir = ( - selector.update_recipe() + selector.update_recipe(quiet=True) ) show_update_recipe_results( stdscr, diff --git a/ignored_files.yml b/ignored_files.yml index cebc19c..acd4a6a 100644 --- a/ignored_files.yml +++ b/ignored_files.yml @@ -4,6 +4,15 @@ field: field: name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__ownership reason: because we need target id instead of uuid +field: + name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__custody + reason: because we need target id instead of uuid +field: + name: field.field.wisski_individual.bb48a22d36f1c15cd16b143d23466812.field__object__vf__stay + reason: because we need target id instead of uuid views: name: views.view.block_content reason: false positive +language: + name: language.entity.de + reason: because the inital page setup was german \ No newline at end of file diff --git a/translatable_config_prefixes.json b/translatable_config_prefixes.json new file mode 100644 index 0000000..14800cc --- /dev/null +++ b/translatable_config_prefixes.json @@ -0,0 +1,10 @@ +{ + "prefixes": [ + "core.entity_form_display", + "core.entity_form_mode", + "field.field.wisski_individual", + "field.storage.wisski_individual", + "views.view", + "wisski_core.wisski_bundle" + ] +}