Coverage for .venv/lib/python3.13/site-packages/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py: 81%

1""" 

2Prebuilt regex patterns for content filtering. 

3 

4This module loads predefined regex patterns from patterns.json for detecting 

5sensitive information like SSNs, credit cards, API keys, etc. 

6""" 

7 

8import json 

9import os 

10import re 

11from enum import Enum 

12from re import Pattern 

13from typing import Any, Final 

14 

15 

16def _load_patterns_from_json() -> dict: 

17 """Load pattern definitions from patterns.json file""" 

18 json_path: Final = os.path.join(os.path.dirname(__file__), "patterns.json") 

19 with open(json_path, "r") as f: 

20 return json.load(f) 

21 

22 

23# Load patterns from JSON 

24_PATTERNS_DATA: Final = _load_patterns_from_json() 

25 

26 

27class PrebuiltPatternName(str, Enum): 

28 """Enum for prebuilt pattern names - dynamically generated from JSON""" 

29 

30 

31# Dynamically create enum values from JSON 

32for pattern_data in _PATTERNS_DATA["patterns"]: 

33 setattr(PrebuiltPatternName, pattern_data["name"].upper(), pattern_data["name"]) 

34 

35 

36# Build lookup dictionaries from JSON 

37PREBUILT_PATTERNS: Final[dict[str, str]] = { 

38 pattern_data["name"]: pattern_data["pattern"] for pattern_data in _PATTERNS_DATA["patterns"] 

39} 

40 

41 

42# Capture any extra configuration declared per pattern (e.g., contextual keywords) 

43KNOWN_PATTERN_KEYS: Final = { 

44 "name", 

45 "display_name", 

46 "pattern", 

47 "category", 

48 "action", 

49 "description", 

50} 

51 

52PATTERN_EXTRA_CONFIG: Final[dict[str, dict[str, Any]]] = {} 

53for pattern_data in _PATTERNS_DATA["patterns"]: 

54 extra_config = {key: value for key, value in pattern_data.items() if key not in KNOWN_PATTERN_KEYS} 

55 PATTERN_EXTRA_CONFIG[pattern_data["name"]] = extra_config 

56 

57 

58def get_compiled_pattern(pattern_name: str) -> Pattern[str]: 

59 """ 

60 Get a compiled regex pattern by name. 

61 

62 Args: 

63 pattern_name: Name of the prebuilt pattern 

64 

65 Returns: 

66 Compiled regex pattern 

67 

68 Raises: 

69 ValueError: If pattern_name is not found in PREBUILT_PATTERNS 

70 """ 

71 if pattern_name not in PREBUILT_PATTERNS: 

72 available_patterns: Final = ", ".join(PREBUILT_PATTERNS.keys()) 

73 raise ValueError(f"Unknown pattern name: '{pattern_name}'. Available patterns: {available_patterns}") 

74 

75 return re.compile(PREBUILT_PATTERNS[pattern_name], re.IGNORECASE) 

76 

77 

78def get_all_pattern_names() -> list[str]: 

79 """ 

80 Get a list of all available prebuilt pattern names. 

81 

82 Returns: 

83 List of pattern names 

84 """ 

85 return list(PREBUILT_PATTERNS.keys()) 

86 

87 

88# Build category mapping from JSON 

89PATTERN_CATEGORIES: Final[dict[str, list[str]]] = {} 

90for pattern_data in _PATTERNS_DATA["patterns"]: 

91 category = pattern_data["category"] 

92 if category not in PATTERN_CATEGORIES: 

93 PATTERN_CATEGORIES[category] = [] 

94 PATTERN_CATEGORIES[category].append(pattern_data["name"]) 

95 

96 

97# Build display names mapping from JSON 

98PATTERN_DISPLAY_NAMES: Final[dict[str, str]] = { 

99 pattern_data["name"]: pattern_data["display_name"] for pattern_data in _PATTERNS_DATA["patterns"] 

100} 

101 

102 

103# Build descriptions mapping from JSON 

104PATTERN_DESCRIPTIONS: Final[dict[str, str]] = { 

105 pattern_data["name"]: pattern_data["description"] for pattern_data in _PATTERNS_DATA["patterns"] 

106} 

107 

108 

109def get_pattern_metadata() -> list[dict[str, str]]: 

110 """ 

111 Return pattern metadata for UI display. 

112 

113 Returns: 

114 List of dictionaries containing pattern name, display_name, category, and description 

115 """ 

116 return [ 

117 { 

118 "name": pattern_data["name"], 

119 "display_name": pattern_data["display_name"], 

120 "category": pattern_data["category"], 

121 "description": pattern_data["description"], 

122 } 

123 for pattern_data in _PATTERNS_DATA["patterns"] 

124 ] 

125 

126 

127def get_available_content_categories() -> list[dict[str, str]]: 

128 """ 

129 Return available content categories for UI display. 

130 

131 Includes categories defined in .yaml/.yml files and in .json files 

132 (e.g. harm_toxic_abuse.json). 

133 

134 Returns: 

135 List of dictionaries containing category name, display_name, and description 

136 """ 

137 import yaml 

138 

139 categories_dir: Final = os.path.join(os.path.dirname(__file__), "categories") 

140 available_categories: Final = [] 

141 

142 if not os.path.exists(categories_dir): 142 ↛ 143line 142 didn't jump to line 143 because the condition on line 142 was never true

143 return [] 

144 

145 # Scan the categories directory for YAML files 

146 for filename in os.listdir(categories_dir): 

147 if filename.endswith(".yaml") or filename.endswith(".yml"): 

148 category_file_path = os.path.join(categories_dir, filename) 

149 try: 

150 with open(category_file_path, "r") as f: 

151 category_data = yaml.safe_load(f) 

152 

153 if category_data and "category_name" in category_data: 153 ↛ 146line 153 didn't jump to line 146 because the condition on line 153 was always true

154 # Use explicit display_name if provided, otherwise auto-generate from category_name 

155 display_name = category_data.get("display_name") or ( 

156 category_data["category_name"].replace("_", " ").title() 

157 ) 

158 

159 available_categories.append( 

160 { 

161 "name": category_data["category_name"], 

162 "display_name": display_name, 

163 "description": category_data.get("description", ""), 

164 "default_action": category_data.get("default_action", "BLOCK"), 

165 } 

166 ) 

167 except Exception as e: 

168 # Skip files that can't be loaded but log the error for debugging 

169 from litellm._logging import verbose_proxy_logger 

170 

171 verbose_proxy_logger.warning("Failed to load category file %s: %s", filename, e) 

172 continue 

173 elif filename.endswith(".json"): 173 ↛ 146line 173 didn't jump to line 146 because the condition on line 173 was always true

174 # JSON category files (e.g. harm_toxic_abuse.json) - no YAML header, use filename 

175 category_name = os.path.splitext(filename)[0] 

176 try: 

177 if category_name == "harm_toxic_abuse": 

178 display_name = "Harmful Toxic Abuse" 

179 description = "Detects harmful, toxic, or abusive language and content" 

180 else: 

181 display_name = category_name.replace("_", " ").title() 

182 description = f"Content category: {display_name}" 

183 available_categories.append( 

184 { 

185 "name": category_name, 

186 "display_name": display_name, 

187 "description": description, 

188 "default_action": "BLOCK", 

189 } 

190 ) 

191 except Exception: 

192 continue 

193 

194 # Sort by name for consistent ordering 

195 available_categories.sort(key=lambda x: x["name"]) 

196 

197 return available_categories