Coverage for utilities/management/commands/rebuild_ltree_paths.py: 0%

1from django.apps import apps 

2from django.core.management.base import BaseCommand, CommandError 

3from django.db import connection, transaction 

4 

5from netbox.models.ltree import LtreeModel 

6from netbox.plugins import PluginConfig 

7from utilities.mptt_to_ltree import ( 

8 count_stale_rows_sql, 

9 populate_paths_sql, 

10 unreachable_rows_sql, 

11) 

12 

13 

14class Command(BaseCommand): 

15 help = ( 

16 "Recompute the trigger-maintained path (and sort_path) columns of hierarchical models " 

17 "from their parent relationships" 

18 ) 

19 

20 # How many offending ids a refusal names. Enough to start from, short enough to read. 

21 REPORTED_IDS = 10 

22 

23 def add_arguments(self, parser): 

24 parser.add_argument( 

25 'model', nargs='*', 

26 help="Limit the rebuild to these models, as app_label.ModelName (default: all)", 

27 ) 

28 parser.add_argument( 

29 '--check', action='store_true', 

30 help="Report which models need rebuilding, without modifying anything", 

31 ) 

32 

33 def get_models(self, names): 

34 """ 

35 Return the concrete core hierarchical models to operate on: those named, in the 

36 order given, or every one of them ordered by table name. 

37 

38 Plugin models are excluded, including when named explicitly: the SQL which rebuilds 

39 `sort_path` reads the name column by name, while `InstallLtreeTriggers` lets a plugin 

40 maintain it from any column, so rebuilding one is not something this command can do 

41 correctly. A plugin in that position needs its own repair path. 

42 """ 

43 def concrete_subclasses(base): 

44 for subclass in base.__subclasses__(): 

45 if subclass._meta.abstract: 

46 yield from concrete_subclasses(subclass) 

47 elif not isinstance(apps.get_app_config(subclass._meta.app_label), PluginConfig): 

48 yield subclass 

49 

50 candidates = { 

51 model._meta.label_lower: model for model in concrete_subclasses(LtreeModel) 

52 } 

53 

54 if not names: 

55 return sorted(candidates.values(), key=lambda model: model._meta.db_table) 

56 

57 models = [] 

58 for name in names: 

59 model = candidates.get(name.lower()) 

60 if model is None: 

61 raise CommandError(f"{name} is not a core hierarchical (ltree-backed) model") 

62 models.append(model) 

63 return models 

64 

65 def check_reachable(self, cursor, model): 

66 """ 

67 Raise unless every row is reachable from a root by following `parent_id`. 

68 

69 The rebuild walks down from `parent_id IS NULL`, so a row no root can reach is one 

70 it silently leaves alone. Reporting success in that case would be the same failure 

71 this command exists to repair: an operation which appears to have worked while the 

72 data is still wrong. Refuse instead, and leave correcting the parent relationships 

73 to the operator, since only they can say what the intended hierarchy was. 

74 

75 Takes the caller's cursor so a refusal rolls back with the transaction the rebuild 

76 would have run in. That does not make the pair atomic with respect to other 

77 writers: under READ COMMITTED every statement takes a fresh snapshot, so a 

78 reparent committed between the check and the rebuild is still missed. Pause writes 

79 for the duration, as the documentation says to. 

80 """ 

81 cursor.execute(unreachable_rows_sql(model._meta.db_table, self.REPORTED_IDS)) 

82 unreachable, ids = cursor.fetchone() 

83 

84 if unreachable: 

85 listed = ', '.join(str(pk) for pk in ids) 

86 if unreachable > len(ids): 

87 listed += ', ...' 

88 raise CommandError( 

89 f'{model._meta.label_lower}: {unreachable} row(s) cannot be reached from a ' 

90 f'root by following parent_id, so a rebuild would skip them: {listed}. ' 

91 f'Correct the parent relationships, then re-run.' 

92 ) 

93 

94 def report_stale(self, model): 

95 """ 

96 Report whether a model's stored paths disagree with its parent relationships. 

97 

98 Read-only, and takes no locks, so it can be run outside a maintenance window or 

99 against a replica. It answers which models need rebuilding, not how many rows are 

100 damaged: see `count_stale_rows_sql()` for why the counts understate a deep tree. 

101 """ 

102 with connection.cursor() as cursor: 

103 cursor.execute( 

104 count_stale_rows_sql(model._meta.db_table, sort_path=model._has_sort_path()) 

105 ) 

106 stale_paths, stale_sort_paths = cursor.fetchone() 

107 

108 if not (stale_paths or stale_sort_paths): 

109 self.stdout.write(f'{model._meta.label_lower}: OK') 

110 return False 

111 

112 damage = [] 

113 if stale_paths: 

114 damage.append(f'{stale_paths} path') 

115 if stale_sort_paths: 

116 damage.append(f'{stale_sort_paths} sort_path') 

117 self.stdout.write(self.style.WARNING( 

118 f"{model._meta.label_lower}: {', '.join(damage)} row(s) out of date" 

119 )) 

120 return True 

121 

122 def handle(self, *args, **options): 

123 models = self.get_models(options['model']) 

124 

125 if options['check']: 

126 stale = [model for model in models if self.report_stale(model)] 

127 if stale: 

128 names = ' '.join(model._meta.label_lower for model in stale) 

129 self.stdout.write(f'\nNeeds rebuilding: {names}') 

130 else: 

131 self.stdout.write(self.style.SUCCESS('Nothing to rebuild.')) 

132 return 

133 

134 # Each table is checked and rebuilt in its own transaction. Tables already done 

135 # stay done if a later one fails or is refused: rebuilding one table cannot leave 

136 # another inconsistent, and holding every table's row locks until the last one 

137 # finished would turn several short blocking windows into one long one. 

138 for model in models: 

139 with transaction.atomic(), connection.cursor() as cursor: 

140 # Announce the rebuild only once the check has passed, so a refusal does 

141 # not print "rebuilding..." for a table left untouched. 

142 self.check_reachable(cursor, model) 

143 self.stdout.write(f'{model._meta.label_lower}: rebuilding... ', ending='') 

144 self.stdout.flush() 

145 # populate_paths_sql() is the same SQL which backfilled these columns 

146 # during the ltree migrations. It relies on SET LOCAL, so it must run 

147 # inside a transaction, and the UPDATE it emits locks every row in the 

148 # table until it commits. 

149 cursor.execute( 

150 populate_paths_sql(model._meta.db_table, sort_path=model._has_sort_path()) 

151 ) 

152 self.stdout.write(self.style.SUCCESS('done')) 

153 

154 self.stdout.write(self.style.SUCCESS('Finished.'))