0
# File Data Sources
1

2
File-based data source implementations providing automatic refresh capabilities for local file systems with configurable buffer sizes, character encodings, and refresh intervals.
3

4
## Capabilities
5

6
### FileRefreshableDataSource Class
7

8
The `FileRefreshableDataSource` class provides file-based readable data source with automatic refresh when the underlying file is modified.
9

10
```java { .api }
11
/**
12
 * A ReadableDataSource based on file. This class will automatically
13
 * fetches the backend file every refresh period when file is modified.
14
 * Limitations: Default read buffer size is 1 MB. If file size is greater than
15
 * buffer size, exceeding bytes will be ignored. Default charset is UTF-8.
16
 * @param <T> target data type
17
 */
18
public class FileRefreshableDataSource<T> extends AutoRefreshDataSource<String, T> {
19
    
20
    // Constants
21
    public static final long DEFAULT_REFRESH_MS = 3000L;
22
    public static final int DEFAULT_BUF_SIZE = 1024 * 1024; // 1 MB
23
    public static final int MAX_SIZE = 1024 * 1024 * 4; // 4 MB
24
    
25
    /**
26
     * Create a file based ReadableDataSource with default settings.
27
     * Buffer size: 1MB, charset: UTF-8, refresh interval: 3 seconds.
28
     * @param file the file to read
29
     * @param configParser the config decoder (parser)
30
     * @throws FileNotFoundException if file doesn't exist or is a directory
31
     */
32
    public FileRefreshableDataSource(File file, Converter<String, T> configParser) 
33
            throws FileNotFoundException;
34
    
35
    /**
36
     * Create a file based ReadableDataSource with filename.
37
     * @param fileName the file name to read
38
     * @param configParser the config decoder (parser)
39
     * @throws FileNotFoundException if file doesn't exist or is a directory
40
     */
41
    public FileRefreshableDataSource(String fileName, Converter<String, T> configParser) 
42
            throws FileNotFoundException;
43
    
44
    /**
45
     * Create a file based ReadableDataSource with custom buffer size.
46
     * @param file the file to read
47
     * @param configParser the config decoder (parser)
48
     * @param bufSize buffer size for reading (must be between 0 and MAX_SIZE)
49
     * @throws FileNotFoundException if file doesn't exist or is a directory
50
     * @throws IllegalArgumentException if bufSize is invalid
51
     */
52
    public FileRefreshableDataSource(File file, Converter<String, T> configParser, int bufSize)
53
            throws FileNotFoundException;
54
    
55
    /**
56
     * Create a file based ReadableDataSource with custom charset.
57
     * @param file the file to read
58
     * @param configParser the config decoder (parser)
59
     * @param charset character encoding for reading the file
60
     * @throws FileNotFoundException if file doesn't exist or is a directory
61
     * @throws IllegalArgumentException if charset is null
62
     */
63
    public FileRefreshableDataSource(File file, Converter<String, T> configParser, Charset charset)
64
            throws FileNotFoundException;
65
    
66
    /**
67
     * Create a file based ReadableDataSource with full customization.
68
     * @param file the file to read
69
     * @param configParser the config decoder (parser)
70
     * @param recommendRefreshMs refresh interval in milliseconds
71
     * @param bufSize buffer size for reading (must be between 0 and MAX_SIZE)
72
     * @param charset character encoding for reading the file
73
     * @throws FileNotFoundException if file doesn't exist or is a directory
74
     * @throws IllegalArgumentException if parameters are invalid
75
     */
76
    public FileRefreshableDataSource(File file, Converter<String, T> configParser, 
77
            long recommendRefreshMs, int bufSize, Charset charset) throws FileNotFoundException;
78
    
79
    /**
80
     * Read file content as string.
81
     * @return file content as string with specified charset
82
     * @throws Exception if file reading fails
83
     */
84
    public String readSource() throws Exception;
85
    
86
    /**
87
     * Check if file was modified since last read.
88
     * @return true if file modification time changed
89
     */
90
    protected boolean isModified();
91
    
92
    /**
93
     * Close data source and clean up resources.
94
     * @throws Exception if cleanup fails
95
     */
96
    public void close() throws Exception;
97
}
98
```
99

100
**Usage Examples:**
101

102
```java
103
// Basic file monitoring with JSON rules
104
Converter<String, List<FlowRule>> jsonConverter = source -> {
105
    if (source == null || source.trim().isEmpty()) {
106
        return new ArrayList<>();
107
    }
108
    return JSON.parseArray(source, FlowRule.class);
109
};
110

111
// Monitor flow rules file
112
FileRefreshableDataSource<List<FlowRule>> flowRuleDs = 
113
    new FileRefreshableDataSource<>(
114
        new File("/sentinel/flow-rules.json"), 
115
        jsonConverter
116
    );
117

118
// Register for automatic rule updates
119
flowRuleDs.getProperty().addListener(rules -> {
120
    FlowRuleManager.loadRules(rules);
121
    System.out.println("Flow rules updated: " + rules.size() + " rules");
122
});
123

124
// Custom buffer size and charset for large configuration files
125
FileRefreshableDataSource<List<DegradeRule>> degradeRuleDs = 
126
    new FileRefreshableDataSource<>(
127
        new File("/sentinel/degrade-rules.xml"),
128
        xmlConverter,
129
        2 * 1024 * 1024, // 2MB buffer
130
        StandardCharsets.UTF_16
131
    );
132

133
// Fast refresh for development
134
FileRefreshableDataSource<List<SystemRule>> devSystemRuleDs = 
135
    new FileRefreshableDataSource<>(
136
        new File("/dev/system-rules.json"),
137
        systemRuleConverter,
138
        1000, // 1 second refresh
139
        DEFAULT_BUF_SIZE,
140
        StandardCharsets.UTF_8
141
    );
142
```
143

144
### FileWritableDataSource Class
145

146
The `FileWritableDataSource` class provides thread-safe file writing capabilities for persisting configuration changes.
147

148
```java { .api }
149
/**
150
 * A WritableDataSource based on file.
151
 * @param <T> data type
152
 */
153
public class FileWritableDataSource<T> implements WritableDataSource<T> {
154
    
155
    /**
156
     * Create a file writable data source with file path.
157
     * @param filePath path to the target file
158
     * @param configEncoder converter from data to string
159
     * @throws IllegalArgumentException if parameters are invalid
160
     */
161
    public FileWritableDataSource(String filePath, Converter<T, String> configEncoder);
162
    
163
    /**
164
     * Create a file writable data source with File object.
165
     * @param file the target file
166
     * @param configEncoder converter from data to string
167
     * @throws IllegalArgumentException if file is null, directory, or configEncoder is null
168
     */
169
    public FileWritableDataSource(File file, Converter<T, String> configEncoder);
170
    
171
    /**
172
     * Create a file writable data source with custom charset.
173
     * @param file the target file
174
     * @param configEncoder converter from data to string
175
     * @param charset character encoding for writing
176
     * @throws IllegalArgumentException if any parameter is null or file is directory
177
     */
178
    public FileWritableDataSource(File file, Converter<T, String> configEncoder, Charset charset);
179
    
180
    /**
181
     * Write value to file in a thread-safe manner.
182
     * @param value value to write
183
     * @throws Exception if file writing fails
184
     */
185
    public void write(T value) throws Exception;
186
    
187
    /**
188
     * Close data source (no-op for file implementation).
189
     * @throws Exception if cleanup fails
190
     */
191
    public void close() throws Exception;
192
}
193
```
194

195
**Usage Examples:**
196

197
```java
198
// JSON encoder for flow rules
199
Converter<List<FlowRule>, String> flowRuleEncoder = rules -> {
200
    return JSON.toJSONString(rules, true); // Pretty print
201
};
202

203
// Writable data source for persisting rule changes
204
FileWritableDataSource<List<FlowRule>> writableDs = 
205
    new FileWritableDataSource<>(
206
        "/sentinel/flow-rules.json", 
207
        flowRuleEncoder
208
    );
209

210
// Write updated rules
211
List<FlowRule> newRules = Arrays.asList(
212
    new FlowRule()
213
        .setResource("GET:/api/users")
214
        .setCount(100)
215
        .setGrade(RuleConstant.FLOW_GRADE_QPS),
216
    new FlowRule()
217
        .setResource("POST:/api/orders")
218
        .setCount(50)
219
        .setGrade(RuleConstant.FLOW_GRADE_QPS)
220
);
221

222
writableDs.write(newRules);
223

224
// XML encoder with custom charset
225
Converter<List<DegradeRule>, String> xmlEncoder = rules -> {
226
    StringBuilder xml = new StringBuilder();
227
    xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
228
    xml.append("<rules>\n");
229
    
230
    for (DegradeRule rule : rules) {
231
        xml.append("  <rule resource=\"").append(rule.getResource()).append("\"")
232
           .append(" count=\"").append(rule.getCount()).append("\"")
233
           .append(" timeWindow=\"").append(rule.getTimeWindow()).append("\"/>\n");
234
    }
235
    
236
    xml.append("</rules>");
237
    return xml.toString();
238
};
239

240
FileWritableDataSource<List<DegradeRule>> xmlWriter = 
241
    new FileWritableDataSource<>(
242
        new File("/config/degrade-rules.xml"),
243
        xmlEncoder,
244
        StandardCharsets.UTF_8
245
    );
246
```
247

248
## Integration Patterns
249

250
### Read-Write Configuration Management
251

252
Combine readable and writable data sources for complete configuration management:
253

254
```java
255
// Setup read and write data sources for the same file
256
File configFile = new File("/sentinel/flow-rules.json");
257

258
Converter<String, List<FlowRule>> reader = source -> 
259
    JSON.parseArray(source, FlowRule.class);
260
    
261
Converter<List<FlowRule>, String> writer = rules -> 
262
    JSON.toJSONString(rules, true);
263

264
// Readable data source with auto-refresh
265
FileRefreshableDataSource<List<FlowRule>> readableDs = 
266
    new FileRefreshableDataSource<>(configFile, reader);
267

268
// Writable data source for updates
269
FileWritableDataSource<List<FlowRule>> writableDs = 
270
    new FileWritableDataSource<>(configFile, writer);
271

272
// Dynamic rule management
273
readableDs.getProperty().addListener(FlowRuleManager::loadRules);
274

275
// API endpoint for rule updates
276
@PostMapping("/flow-rules")
277
public ResponseEntity<String> updateFlowRules(@RequestBody List<FlowRule> rules) {
278
    try {
279
        writableDs.write(rules);
280
        return ResponseEntity.ok("Rules updated successfully");
281
    } catch (Exception e) {
282
        return ResponseEntity.status(500).body("Failed to update rules: " + e.getMessage());
283
    }
284
}
285
```
286

287
### Configuration File Formats
288

289
Support multiple configuration formats through different converters:
290

291
```java
292
// YAML converter
293
Converter<String, List<FlowRule>> yamlConverter = source -> {
294
    Yaml yaml = new Yaml();
295
    Map<String, Object> data = yaml.load(source);
296
    List<Map<String, Object>> rulesData = (List<Map<String, Object>>) data.get("flowRules");
297
    
298
    return rulesData.stream()
299
        .map(ruleData -> new FlowRule()
300
            .setResource((String) ruleData.get("resource"))
301
            .setCount(((Number) ruleData.get("count")).doubleValue()))
302
        .collect(Collectors.toList());
303
};
304

305
// Properties converter
306
Converter<String, List<FlowRule>> propsConverter = source -> {
307
    Properties props = new Properties();
308
    props.load(new StringReader(source));
309
    
310
    List<FlowRule> rules = new ArrayList<>();
311
    for (String key : props.stringPropertyNames()) {
312
        if (key.startsWith("flow.rule.")) {
313
            String resource = key.substring("flow.rule.".length());
314
            double count = Double.parseDouble(props.getProperty(key));
315
            rules.add(new FlowRule().setResource(resource).setCount(count));
316
        }
317
    }
318
    return rules;
319
};
320
```
321

322
## Error Handling and Limitations
323

324
### File Size Limitations
325

326
- **Default buffer size**: 1 MB
327
- **Maximum buffer size**: 4 MB
328
- **Behavior**: Files larger than buffer size will be truncated
329
- **Recommendation**: Use appropriate buffer size for your configuration files
330

331
### Character Encoding
332

333
- **Default charset**: UTF-8
334
- **Custom charset**: Specify in constructor
335
- **Encoding consistency**: Ensure read and write operations use same charset
336

337
### Thread Safety
338

339
- **FileRefreshableDataSource**: Thread-safe for concurrent reads
340
- **FileWritableDataSource**: Uses `ReentrantLock` for thread-safe writes
341
- **File system**: Handles concurrent access through proper locking mechanisms