diff --git a/.gitignore b/.gitignore index f6635e3..cdcc649 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ new_recipe_not_in_original.csv deleted_files.csv ignored_files.yml recipes/* +update/* +.venv +language_inconsistency.csv \ No newline at end of file diff --git a/README.md b/README.md index 64f627f..6b7a38a 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,31 @@ This script helps you manage and transfer specific Drupal configuration files by - Python 3.6+ - curses library (built into most Python installations on Linux/Mac) -- PyYAML library (install with: `pip install pyyaml`) +- PyYAML (`pyyaml`) ## Installation and Setup +From the `configure_man/` directory: + +```bash +cd configure_man + +# Create a virtual environment +python3 -m venv .venv + +# Activate it (Linux/macOS) +source .venv/bin/activate + +# On Windows (PowerShell) +# .venv\Scripts\Activate.ps1 + +# Install dependencies +pip install --upgrade pip +pip install -r requirements.txt +``` + +To run the tool later, activate the venv again (`source .venv/bin/activate`) before calling `python3 config_selector.py`. + The script operates in the current working directory where you run it. It will create the following subdirectories if they don't exist: - `ingest/` - Drop your exported Drupal config archive here (named `config*.tar.gz`) - `original_config/` - YAML files are extracted here automatically from the ingest archive diff --git a/config_selector.py b/config_selector.py index 1cc5bf3..92ce2b0 100644 --- a/config_selector.py +++ b/config_selector.py @@ -19,6 +19,13 @@ BASE_DIR = os.getcwd() INGEST_DIR = os.path.join(BASE_DIR, "ingest") ORIGINAL_CONFIG_DIR = os.path.join(BASE_DIR, "original_config") RECIPE_CONFIG_DIR = os.path.join(BASE_DIR, "recipes", "wisski_default_data_model", "config") +RECIPE_SOURCE_DIR = os.path.join(BASE_DIR, "recipes", "wisski_default_data_model") +RECIPE_UPDATE_BASE = os.path.join(BASE_DIR, "update") + + +def recipe_update_dir(recipe_source: str = RECIPE_SOURCE_DIR) -> str: + """Path to updated recipe copy: update//.""" + return os.path.join(RECIPE_UPDATE_BASE, os.path.basename(recipe_source)) def _clear_dir(path: str) -> int: @@ -36,6 +43,13 @@ def _clear_dir(path: str) -> int: return removedCount +def _copy_recipe_tree(src: str, dst: str) -> None: + """Copy a full recipe directory tree, including .git.""" + if os.path.exists(dst): + shutil.rmtree(dst) + shutil.copytree(src, dst) + + def ingest_config(force: bool = False) -> None: """Extract a config*.tar.gz archive from ingest/ into original_config/. @@ -103,6 +117,7 @@ class ConfigSelector: self.changed_files_path = changed_files_path or os.path.join(BASE_DIR, "changed_files") self.deleted_files_csv = deleted_files_csv or os.path.join(BASE_DIR, "deleted_files.csv") self.new_recipe_not_in_original_csv = os.path.join(BASE_DIR, "new_recipe_not_in_original.csv") + self.language_inconsistency_csv = os.path.join(BASE_DIR, "language_inconsistency.csv") self.changed_files_manifest_csv = os.path.join(BASE_DIR, "changed_files_manifest.csv") self.ignored_files_yml = ignored_files_yml or os.path.join(BASE_DIR, "ignored_files.yml") self.prefixes: List[str] = [] @@ -404,7 +419,107 @@ class ConfigSelector: """Join config root with a posix rel_path from _collect_yaml_relpaths.""" return os.path.normpath(os.path.join(config_root, *rel_path.split("/"))) - def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int]: + def _collect_language_inconsistencies(self) -> List[str]: + """Language overrides in config_path with no matching base *.yml at config root.""" + inconsistent: List[str] = [] + lang_root = os.path.join(self.config_path, "language") + if not os.path.isdir(lang_root): + return inconsistent + try: + for lang_code in os.listdir(lang_root): + lang_dir = os.path.join(lang_root, lang_code) + if not os.path.isdir(lang_dir): + continue + for name in os.listdir(lang_dir): + if not name.endswith(".yml"): + continue + base_path = os.path.join(self.config_path, name) + if not os.path.isfile(base_path): + inconsistent.append(f"language/{lang_code}/{name}") + except OSError: + pass + return sorted(inconsistent) + + def _write_language_inconsistency_csv(self, inconsistent: List[str]) -> None: + with open(self.language_inconsistency_csv, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["filename"]) + for rel_path in inconsistent: + writer.writerow([rel_path]) + + def delete_language_inconsistencies(self) -> int: + """Remove language/*/*.yml in config_path that have no base config pendant.""" + inconsistent = self._collect_language_inconsistencies() + removed = 0 + for rel_path in inconsistent: + path = self._abs_config_path(self.config_path, rel_path) + try: + if os.path.isfile(path): + os.remove(path) + removed += 1 + except OSError: + pass + self._remove_empty_language_dirs(self.config_path) + remaining = self._collect_language_inconsistencies() + self._write_language_inconsistency_csv(remaining) + return removed + + @staticmethod + def _read_deleted_files_csv(csv_path: str) -> List[str]: + paths: List[str] = [] + if not os.path.isfile(csv_path): + return paths + with open(csv_path, newline="") as csvfile: + reader = csv.DictReader(csvfile) + for row in reader: + name = (row.get("filename") or "").strip() + if name: + paths.append(name) + return paths + + def update_recipe(self) -> Tuple[int, int, int, str]: + """Copy committed recipe to update//, apply deletions and changed_files/. + + Returns: + Tuple of (files_deleted, files_applied, deletes_missing, update_dir). + """ + if not os.path.isdir(RECIPE_SOURCE_DIR): + raise FileNotFoundError(f"Recipe source not found: {RECIPE_SOURCE_DIR}") + + update_dir = recipe_update_dir() + _copy_recipe_tree(RECIPE_SOURCE_DIR, update_dir) + update_config = os.path.join(update_dir, "config") + + files_deleted = 0 + deletes_missing = 0 + for rel_path in self._read_deleted_files_csv(self.deleted_files_csv): + target = self._abs_config_path(update_config, rel_path) + try: + if os.path.isfile(target): + os.remove(target) + files_deleted += 1 + else: + deletes_missing += 1 + except OSError: + deletes_missing += 1 + self._remove_empty_language_dirs(update_config) + + files_applied = 0 + for rel_path in sorted(self._collect_yaml_relpaths(self.changed_files_path)): + src = self._abs_config_path(self.changed_files_path, rel_path) + dst = self._abs_config_path(update_config, rel_path) + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + files_applied += 1 + + print( + f"Updated recipe written to {update_dir}/ " + f"({files_deleted} deleted from config, {files_applied} copied from changed_files/, " + f"{deletes_missing} delete entries not present in copy)." + ) + return files_deleted, files_applied, deletes_missing, update_dir + + def compare_and_track_changes(self) -> Tuple[int, int, int, int, int, int, int]: """ Compare committed recipe config (old_recipe_path) to new_recipe_config (config_path). @@ -414,10 +529,12 @@ class ConfigSelector: - Writes deleted_files.csv: paths in committed recipe missing from new_recipe_config. - Writes changed_files_manifest.csv: relative_path + kind (new|modified) for each copy. - Writes new_recipe_not_in_original.csv for paths in new_recipe not in original export. + - Writes language_inconsistency.csv for language/*/*.yml without a base config pendant. Returns: Tuple of (added_count, modified_count, deleted_count, unchanged_count, - language_deleted_count, not_in_original_count) + language_deleted_count, not_in_original_count, + language_inconsistency_count) """ _clear_dir(self.changed_files_path) os.makedirs(self.changed_files_path, exist_ok=True) @@ -523,6 +640,16 @@ class ConfigSelector: elif not orig_paths: print("Warning: original_config has no YAML; orphan CSV is empty.") + language_inconsistent = self._collect_language_inconsistencies() + language_inconsistency_count = len(language_inconsistent) + self._write_language_inconsistency_csv(language_inconsistent) + if language_inconsistency_count: + print( + f"{language_inconsistency_count} language override(s) in new_recipe_config " + f"have no base config pendant " + f"(see {os.path.basename(self.language_inconsistency_csv)})." + ) + return ( added_count, modified_count, @@ -530,6 +657,7 @@ class ConfigSelector: unchanged_count, language_deleted_count, not_in_original_count, + language_inconsistency_count, ) def add_prefix(self, prefix: str) -> None: @@ -715,7 +843,11 @@ def draw_menu(stdscr, selector: ConfigSelector, selected_idx: int, view_mode: st # Instructions stdscr.addstr(h-4, 2, "LEFT/RIGHT: Navigate tree levels | SPACE: Toggle view | ENTER: Select files") - stdscr.addstr(h-3, 2, "A: Add prefix | O: Choose | D: Add/Del | C: Copy | M: Compare | S: Save | Q: Quit") + stdscr.addstr( + h - 3, + 2, + "A: Add | O: Choose | D: Add/Del | C: Copy | M: Compare | I: Lang orphans | U: Update | S: Save | Q: Quit", + ) # Content area start_y = 3 @@ -916,7 +1048,6 @@ def confirm_dialog(stdscr, message): dialog_win.addstr(3, 2, "Press Y to confirm, any other key to cancel") dialog_win.refresh() - # Get input key = stdscr.getch() return key == ord('y') or key == ord('Y') @@ -1295,6 +1426,78 @@ def confirm_yaml_cleaning(stdscr, selector): return confirm_dialog(stdscr, "Copy all matched files to config directory and remove UUID and _core?") +def _curses_addstr_clipped(win, y: int, x: int, text: str, max_width: int, attrs: int = 0) -> None: + """Write text inside a bordered curses window, clipping to fit.""" + win_h, _ = win.getmaxyx() + if y <= 0 or y >= win_h - 1: + return + clipped = text[: max(0, max_width - x - 1)] + if not clipped: + return + try: + if attrs: + win.addstr(y, x, clipped, attrs) + else: + win.addstr(y, x, clipped) + except curses.error: + pass + +def show_info_dialog(stdscr, message: str) -> None: + """Show a simple message dialog.""" + h, w = stdscr.getmaxyx() + if h < 5 or w < 20: + return + dialog_h = min(5, h - 2) + dialog_w = min(max(40, len(message) + 4), w - 4) + dialog_y = max(0, (h - dialog_h) // 2) + dialog_x = max(0, (w - dialog_w) // 2) + if dialog_h <= 0 or dialog_w <= 0: + return + dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) + dialog_win.box() + _curses_addstr_clipped(dialog_win, 1, 2, message, dialog_w) + _curses_addstr_clipped(dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w) + dialog_win.refresh() + stdscr.getch() + +def show_update_recipe_results( + stdscr, + files_deleted: int, + files_applied: int, + deletes_missing: int, + update_dir: str, +) -> None: + """Show the results of building update// from the committed recipe.""" + h, w = stdscr.getmaxyx() + dialog_w = min(60, w - 4) + content_lines = [ + f"Recipe copy: {update_dir}/", + f"Deleted from config: {files_deleted}", + f"Applied from changed_files/: {files_applied}", + ] + if deletes_missing: + content_lines.append(f"Delete CSV entries not in copy: {deletes_missing}") + content_start = 3 + dialog_h = min(h - 2, max(12, content_start + len(content_lines) + 3)) + dialog_y = max(0, (h - dialog_h) // 2) + dialog_x = max(0, (w - dialog_w) // 2) + dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) + dialog_win.box() + _curses_addstr_clipped( + dialog_win, 1, 2, "Recipe Update Results", dialog_w, curses.A_BOLD + ) + max_footer_row = dialog_h - 3 + for i, line in enumerate(content_lines): + row = content_start + i + if row > max_footer_row: + break + _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w) + _curses_addstr_clipped( + dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w + ) + dialog_win.refresh() + stdscr.getch() + def show_comparison_results( stdscr, added_count: int, @@ -1303,59 +1506,65 @@ def show_comparison_results( unchanged_count: int, language_deleted_count: int = 0, not_in_original_count: int = 0, + language_inconsistency_count: int = 0, ): """Show the results of file comparison.""" h, w = stdscr.getmaxyx() - dialog_w = 60 + dialog_w = min(60, w - 4) total_out = added_count + modified_count - # Rows from 3: 5 stats + optional notes + blank + 2 CSV hints; footer at dialog_h - 2 - lines = ( - 5 - + (1 if not_in_original_count else 0) - + (1 if language_deleted_count else 0) - + 1 - + 2 - ) - last_content_row = 2 + lines - dialog_h = min(h - 2, max(14, last_content_row + 3)) - dialog_y = (h - dialog_h) // 2 - dialog_x = (w - dialog_w) // 2 - - # Draw dialog box - dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) - dialog_win.box() - dialog_win.addstr(1, 2, "File Comparison Results", curses.A_BOLD) - row = 3 - dialog_win.addstr(row, 4, f"New (not in committed recipe): {added_count}"[: dialog_w - 6]) - row += 1 - dialog_win.addstr(row, 4, f"Modified vs committed: {modified_count}"[: dialog_w - 6]) - row += 1 - dialog_win.addstr(row, 4, f"Total → changed_files/: {total_out}"[: dialog_w - 6]) - row += 1 deleted_line = f"Deleted (recipe, missing in new): {deleted_count}" if language_deleted_count: deleted_line += f" ({language_deleted_count} language/)" - dialog_win.addstr(row, 4, deleted_line[: dialog_w - 6]) - row += 1 - dialog_win.addstr(row, 4, f"Unchanged: {unchanged_count}"[: dialog_w - 6]) - row += 1 + + content_lines = [ + f"New (not in committed recipe): {added_count}", + f"Modified vs committed: {modified_count}", + f"Total -> changed_files/: {total_out}", + deleted_line, + f"Unchanged: {unchanged_count}", + ] if not_in_original_count: - dialog_win.addstr( - row, - 4, - f"Not in original export: {not_in_original_count} (new_recipe_not_in_original.csv)"[ - : dialog_w - 6 - ], + content_lines.append( + f"Not in original export: {not_in_original_count} " + "(new_recipe_not_in_original.csv)" ) - row += 1 if language_deleted_count: - dialog_win.addstr(row, 4, "Language deletions: paths under language/… in deleted CSV") - row += 1 - row += 1 - dialog_win.addstr(row, 4, "Manifest → changed_files_manifest.csv") - row += 1 - dialog_win.addstr(row, 4, "Deletions → deleted_files.csv") - dialog_win.addstr(dialog_h - 2, 2, "Press any key to continue...") + content_lines.append( + "Language deletions: paths under language/ in deleted CSV" + ) + if language_inconsistency_count: + content_lines.append( + f"Language inconsistency: {language_inconsistency_count} " + "(language_inconsistency.csv)" + ) + content_lines.extend( + [ + "", + "Manifest -> changed_files_manifest.csv", + "Deletions -> deleted_files.csv", + ] + ) + + content_start = 3 + # Leave one blank row between content and footer (footer at dialog_h - 2). + dialog_h = min(h - 2, max(14, content_start + len(content_lines) + 3)) + dialog_y = max(0, (h - dialog_h) // 2) + dialog_x = max(0, (w - dialog_w) // 2) + + dialog_win = curses.newwin(dialog_h, dialog_w, dialog_y, dialog_x) + dialog_win.box() + _curses_addstr_clipped( + dialog_win, 1, 2, "File Comparison Results", dialog_w, curses.A_BOLD + ) + max_footer_row = dialog_h - 3 + for i, line in enumerate(content_lines): + row = content_start + i + if row > max_footer_row: + break + _curses_addstr_clipped(dialog_win, row, 4, line, dialog_w) + _curses_addstr_clipped( + dialog_win, dialog_h - 2, 2, "Press any key to continue...", dialog_w + ) dialog_win.refresh() # Wait for key @@ -1530,6 +1739,7 @@ def main(stdscr): unchanged_count, language_deleted_count, not_in_original_count, + language_inconsistency_count, ) = selector.compare_and_track_changes() show_comparison_results( stdscr, @@ -1539,7 +1749,40 @@ def main(stdscr): unchanged_count, language_deleted_count, not_in_original_count, + language_inconsistency_count, ) + elif key == ord('i') or key == ord('I'): + inconsistent = selector._collect_language_inconsistencies() + if not inconsistent: + show_info_dialog(stdscr, "No language inconsistencies in new_recipe_config.") + elif confirm_dialog( + stdscr, + f"Delete {len(inconsistent)} language file(s) without base config pendant?", + ): + removed = selector.delete_language_inconsistencies() + show_info_dialog( + stdscr, + f"Removed {removed} language inconsistency file(s) from new_recipe_config.", + ) + elif key == ord('u') or key == ord('U'): + if confirm_dialog( + stdscr, + f"Copy recipe to {recipe_update_dir()}/, remove deleted_files.csv " + "entries, apply changed_files/ to config?", + ): + try: + files_deleted, files_applied, deletes_missing, update_dir = ( + selector.update_recipe() + ) + show_update_recipe_results( + stdscr, + files_deleted, + files_applied, + deletes_missing, + update_dir, + ) + except FileNotFoundError as exc: + show_info_dialog(stdscr, str(exc)) elif key == ord('s') or key == ord('S'): selector.save_prefixes() elif key in (10, 13): # ENTER key to select individual files diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3aecde9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pyyaml>=6.0