Coverage for /usr/local/lib/python3.10/site-packages/opal_common-0.0.0-py3.10.egg/opal_common/git_utils/bundle_maker.py: 11%

1from functools import partial 

2from pathlib import Path 

3from typing import List, Optional, Set 

4 

5from ddtrace import tracer 

6from git import Repo 

7from git.objects import Commit 

8from opal_common.engine import get_rego_package, is_data_module, is_policy_module 

9from opal_common.git_utils.commit_viewer import ( 

10 CommitViewer, 

11 VersionedDirectory, 

12 VersionedFile, 

13 find_ignore_match, 

14 has_extension, 

15 is_under_directories, 

16) 

17from opal_common.git_utils.diff_viewer import ( 

18 DiffViewer, 

19 diffed_file_has_extension, 

20 diffed_file_is_under_directories, 

21) 

22from opal_common.logger import logger 

23from opal_common.paths import PathUtils 

24from opal_common.schemas.policy import ( 

25 DataModule, 

26 DeletedFiles, 

27 PolicyBundle, 

28 RegoModule, 

29) 

30 

31 

32class BundleMaker: 

33 """creates a policy bundle based on: 

34 

35 - the current state of the policy git repo 

36 - filtering criteria on the policy git repo (specific directories, specific file types, etc) 

37 

38 there are two types of bundles: 

39 - a full/complete bundle, representing the state of the repo at one commit 

40 - a diff bundle, representing only the *changes* made to the policy between two commits (the diff). 

41 """ 

42 

43 def __init__( 

44 self, 

45 repo: Repo, 

46 in_directories: Set[Path], 

47 extensions: Optional[List[str]] = None, 

48 root_manifest_path: str = ".manifest", 

49 bundle_ignore: Optional[List[str]] = None, 

50 ): 

51 """[summary] 

52 

53 Args: 

54 repo (Repo): the policy repo 

55 in_directories (Set[Path]): the directories in the repo that we want to filter on. 

56 if the entire repo is relevant, pass Path(".") as the directory 

57 (all paths are relative to the repo root). 

58 extensions (Optional[List[str]]): optional filtering on file extensions. 

59 bundle_ignore (Optional[List[str]]): optional ignoring of files using glob paths. 

60 Note that the std lib's implementation of Path does not support interpreting double asterisks (**) 

61 in glob paths as recursive directories so globs will need to explicitly match those directories. 

62 Issue: https://github.com/python/cpython/pull/101398 

63 """ 

64 self._repo = repo 

65 self._directories = in_directories 

66 self._has_extension = partial(has_extension, extensions=extensions) 

67 self._is_under_directories = partial( 

68 is_under_directories, directories=in_directories 

69 ) 

70 self._diffed_file_has_extension = partial( 

71 diffed_file_has_extension, extensions=extensions 

72 ) 

73 self._diffed_file_is_under_directories = partial( 

74 diffed_file_is_under_directories, directories=in_directories 

75 ) 

76 self._root_manifest_path = Path(root_manifest_path) 

77 

78 self._bundle_ignore = bundle_ignore 

79 self._find_ignore_match = partial( 

80 find_ignore_match, bundle_ignore=bundle_ignore 

81 ) 

82 self._diffed_file_find_ignore_match = lambda diff: find_ignore_match( 

83 diff.b_path, bundle_ignore 

84 ) 

85 

86 def _get_explicit_manifest(self, viewer: CommitViewer) -> Optional[List[str]]: 

87 """Rego policies often have dependencies (import statements) between 

88 policies. Since the OPAL client is limited by the OPA REST api and this 

89 api currently does not allow to load bundles (multiple policies 

90 together), OPAL client can only load one policy at a time. 

91 

92 If policies with dependencies between them are loaded out-of- 

93 order, OPA will throw an exception. 

94 

95 To mitigate this, we allow the developer to put a manifest file 

96 in the repository (default path: .manifest). This file, if 

97 exists, should contain the list of policy files (.rego) in the 

98 correct order they should be loaded into OPA. 

99 

100 This method searches for an explicit manifest file, reads it and 

101 returns the list of paths, or None if not found. 

102 

103 The manifest file can include references to other directories 

104 containing a ".manifest" file, those would be recursively 

105 expanded to compile the final manifest list. 

106 """ 

107 visited_paths = [] 

108 

109 def _compile_manifest_file( 

110 dir: VersionedDirectory, 

111 manifest_file_name: str = ".manifest", 

112 _branch: List[str] = [], 

113 ) -> List[str]: 

114 explicit_manifest: List[Path] = [] 

115 manifest_file_path = dir.path / manifest_file_name 

116 _branch.append(str(manifest_file_path)) 

117 

118 logger.debug(f"Compiling manifest file { ' -> '.join(_branch)}") 

119 try: 

120 manifest_file = viewer.get_file(dir.path / manifest_file_name) 

121 if manifest_file is None: 

122 logger.info( 

123 f"Manifest file {manifest_file_path} not found, assuming empty" 

124 ) 

125 else: 

126 for path_entry in manifest_file.read().splitlines(): 

127 # Path is relative to current directory, make it absolute 

128 path_entry = dir.path / path_entry 

129 

130 if ( 

131 path_entry.is_absolute() 

132 or dir.path.resolve() not in path_entry.resolve().parents 

133 ): 

134 # Block absolute paths or paths with ".." (CommitViewer ignores those anyway, but be explicit) 

135 logger.warning( 

136 f" Path '{path_entry}' is outside current .manifest directory" 

137 ) 

138 continue 

139 

140 if not viewer.exists(path_entry): 

141 logger.warning(f" Path '{path_entry}' does not exist") 

142 continue 

143 

144 ignore_path_match = find_ignore_match( 

145 Path(path_entry), self._bundle_ignore 

146 ) 

147 if ignore_path_match != None: 

148 logger.warning( 

149 f" Path'{path_entry} is ignored by ignore glob '{ignore_path_match}'" 

150 ) 

151 continue 

152 

153 if path_entry in visited_paths: 

154 logger.warning( 

155 f" Path '{path_entry}' has redundant references" 

156 ) 

157 continue 

158 

159 visited_paths.append(path_entry) 

160 

161 dir_entry = viewer.get_directory(path_entry) 

162 if dir_entry is not None: 

163 # Reference to another directory, try to recursively load its manifest file 

164 explicit_manifest += _compile_manifest_file( 

165 dir_entry, _branch=list(_branch) 

166 ) 

167 continue 

168 

169 # This is an existing file 

170 explicit_manifest.append(str(path_entry)) 

171 logger.debug( 

172 f" Path '{path_entry}' was added to explicit manifest" 

173 ) 

174 

175 except Exception as e: 

176 logger.exception( 

177 f" Failed to compile manifest file '{manifest_file_path}'" 

178 ) 

179 return [] 

180 

181 return explicit_manifest 

182 

183 root_manifest = viewer.get_node(self._root_manifest_path) 

184 if isinstance(root_manifest, VersionedFile): 

185 # Root manifest is supplied in old-fashioned way (as file path) - support for backward compatibility 

186 logger.info( 

187 f"Using root manifest file path (old-fashioned): '{root_manifest.path}'" 

188 ) 

189 return _compile_manifest_file( 

190 viewer.get_directory(root_manifest.path.parent), 

191 manifest_file_name=root_manifest.path.name, 

192 ) 

193 

194 elif isinstance(root_manifest, VersionedDirectory): 

195 # Root manifest is supplied in new-fashioned way (as a directory path containing ".manifest" file) 

196 logger.info( 

197 f"Using root manifest dir path (new-fashioned): '{root_manifest.path}'" 

198 ) 

199 return _compile_manifest_file(root_manifest) 

200 

201 else: 

202 logger.info( 

203 f"Root manifest path doesn't exist, no explicit order would be imposed on policy bundle" 

204 ) 

205 return list() 

206 

207 def _sort_manifest( 

208 self, unsorted_manifest: List[str], explicit_sorting: Optional[List[str]] 

209 ) -> List[str]: 

210 """The way this sorting works, is assuming that explicit_sorting does 

211 NOT necessarily contains all the policies found in the manifest, or 

212 that even "policies" mentioned in it actually exists in the actual 

213 generated manifest. 

214 

215 We must ensure that all items in unsorted_manifest must also 

216 exist in the output list. 

217 """ 

218 if not explicit_sorting: 

219 return unsorted_manifest 

220 

221 # casting to Path 

222 unsorted_paths = [Path(path) for path in unsorted_manifest] 

223 sorting = [Path(path) for path in explicit_sorting] 

224 

225 # sorting the list 

226 sorted_paths = PathUtils.sort_paths_according_to_explicit_sorting( 

227 unsorted_paths, sorting 

228 ) 

229 

230 # cast back to string paths 

231 return [str(path) for path in sorted_paths] 

232 

233 def make_bundle(self, commit: Commit) -> PolicyBundle: 

234 """Creates a *complete* bundle of all the policy and data modules found 

235 in the policy repo, when the repo HEAD is at the given `commit`. 

236 

237 Args: 

238 commit (Commit): the commit the repo should be checked out on to search for policy files. 

239 

240 Returns: 

241 bundle (PolicyBundle): the bundle of policy modules found in the repo (checked out on `commit`) 

242 """ 

243 data_modules = [] 

244 policy_modules = [] 

245 manifest = [] 

246 

247 with CommitViewer(commit) as viewer: 

248 filter = ( 

249 lambda f: self._has_extension(f) 

250 and self._is_under_directories(f) 

251 and self._find_ignore_match(f.path) == None 

252 ) 

253 explicit_manifest = self._get_explicit_manifest(viewer) 

254 logger.debug(f"Explicit manifest to be used: {explicit_manifest}") 

255 

256 for source_file in viewer.files(filter): 

257 with tracer.trace( 

258 "bundle_maker.git_file_read", resource=str(source_file.path) 

259 ): 

260 contents = source_file.read() 

261 path = source_file.path 

262 

263 if is_data_module(path): 

264 data_modules.append( 

265 DataModule(path=str(path.parent), data=contents) 

266 ) 

267 manifest.append(str(path)) 

268 elif is_policy_module(path): 

269 policy_modules.append( 

270 RegoModule( 

271 path=str(path), 

272 package_name=get_rego_package(contents) or "", 

273 rego=contents, 

274 ) 

275 ) 

276 manifest.append(str(path)) 

277 

278 return PolicyBundle( 

279 manifest=self._sort_manifest(manifest, explicit_manifest), 

280 hash=commit.hexsha, 

281 data_modules=data_modules, 

282 policy_modules=policy_modules, 

283 ) 

284 

285 def make_diff_bundle(self, old_commit: Commit, new_commit: Commit) -> PolicyBundle: 

286 """Creates a *diff* bundle of all the policy and data modules that were 

287 changed (either added, renamed, modified or deleted) between the 

288 `old_commit` and `new_commit`. essentially all the relevant files when 

289 running `git diff old_commit..new_commit`. 

290 

291 Note that we still filter only directories and file types given in the constructor. 

292 

293 Args: 

294 old_commit (Commit): represents the previous known state of the repo. 

295 The opal client subscribes to the policy state and gets updates from 

296 the server via a pubsub channel. When it receives an update that 

297 new state is available in the policy repo, the client requests a 

298 *diff bundle* from the /policy api route. The client will report 

299 its last known commit as the `old_commit`, and only new state 

300 (the diff from the client known commit to the server newest commit) 

301 will be returned back. 

302 commit (Commit): represents the newest known commit in the server (the new state). 

303 

304 Returns: 

305 bundle (PolicyBundle): a diff bundle containing only the policy modules changed 

306 between `old_commit` and `new_commit`. 

307 """ 

308 data_modules = [] 

309 policy_modules = [] 

310 deleted_data_modules = [] 

311 deleted_policy_modules = [] 

312 manifest = [] 

313 explicit_manifest = self._get_explicit_manifest(CommitViewer(new_commit)) 

314 

315 with DiffViewer(old_commit, new_commit) as viewer: 

316 filter = lambda diff: ( 

317 self._diffed_file_has_extension(diff) 

318 and self._diffed_file_is_under_directories(diff) 

319 and self._diffed_file_find_ignore_match(diff) == None 

320 ) 

321 for source_file in viewer.added_or_modified_files(filter): 

322 contents = source_file.read() 

323 path = source_file.path 

324 

325 if is_data_module(path): 

326 data_modules.append( 

327 # in OPA, the data module path is the containing directory 

328 # i.e: /path/to/data.json will put the json contents under "/path/to" in the opa tree 

329 DataModule(path=str(path.parent), data=contents) 

330 ) 

331 manifest.append(str(path)) 

332 elif is_policy_module(path): 

333 policy_modules.append( 

334 RegoModule( 

335 path=str(path), 

336 package_name=get_rego_package(contents) or "", 

337 rego=contents, 

338 ) 

339 ) 

340 manifest.append(str(path)) 

341 

342 for source_file in viewer.deleted_files(filter): 

343 path = source_file.path 

344 

345 if is_data_module(path): 

346 # in OPA, the data module path is the containing directory (see above) 

347 deleted_data_modules.append(str(path.parent)) 

348 elif is_policy_module(path): 

349 deleted_policy_modules.append(path) 

350 

351 if deleted_data_modules or deleted_policy_modules: 

352 deleted_policies = self._sort_manifest( 

353 deleted_policy_modules, explicit_manifest 

354 ) 

355 deleted_policies.reverse() # when removed, dependent policies should be removed first 

356 

357 deleted_files = DeletedFiles( 

358 data_modules=deleted_data_modules, 

359 policy_modules=deleted_policies, 

360 ) 

361 else: 

362 deleted_files = None 

363 

364 return PolicyBundle( 

365 manifest=self._sort_manifest(manifest, explicit_manifest), 

366 hash=new_commit.hexsha, 

367 old_hash=old_commit.hexsha, 

368 data_modules=data_modules, 

369 policy_modules=policy_modules, 

370 deleted_files=deleted_files, 

371 )