0
# Metadata Management
1

2
The metadata management system provides comprehensive operations for managing tables, partitions, namespaces, and data commits in the LakeSoul lakehouse framework. The central `DBManager` class serves as the primary interface for all metadata operations.
3

4
## Capabilities
5

6
### DBManager Class
7

8
The main entry point for all metadata operations, providing high-level methods for table lifecycle, partition management, and data commit operations.
9

10
```java { .api }
11
/**
12
 * Central metadata management class providing high-level operations
13
 * for tables, partitions, and data commits
14
 */
15
public class DBManager {
16
    /** 
17
     * Create new DBManager instance with default DAO configuration
18
     */
19
    public DBManager();
20
}
21
```
22

23
### Table Operations
24

25
Comprehensive table management including creation, querying, listing, and deletion operations.
26

27
```java { .api }
28
/**
29
 * Check if table exists by table path
30
 * @param tablePath Full path to the table
31
 * @return true if table exists, false otherwise
32
 */
33
public boolean isTableExists(String tablePath);
34

35
/**
36
 * Check if table exists by table name and namespace
37
 * @param tableName Short table name
38
 * @param tableNamespace Table namespace (defaults to "default" if null)
39
 * @return true if table exists, false otherwise
40
 */
41
public boolean isTableExistsByTableName(String tableName, String tableNamespace);
42

43
/**
44
 * Check if table exists by table name in default namespace
45
 * @param tableName Short table name
46
 * @return true if table exists, false otherwise
47
 */
48
public boolean isTableExistsByTableName(String tableName);
49

50
/**
51
 * Check if specific table ID exists for given table path
52
 * @param tablePath Full path to the table
53
 * @param tableId Unique table identifier
54
 * @return true if table ID exists for path, false otherwise
55
 */
56
public boolean isTableIdExists(String tablePath, String tableId);
57

58
/**
59
 * Create a new table with comprehensive metadata
60
 * @param tableId Unique table identifier
61
 * @param namespace Table namespace
62
 * @param tableName Short table name (can be empty)
63
 * @param tablePath Full table storage path
64
 * @param tableSchema JSON schema definition
65
 * @param properties Table properties as JSONObject
66
 * @param partitions Partition configuration string
67
 */
68
public void createNewTable(String tableId, String namespace, String tableName, 
69
                          String tablePath, String tableSchema, JSONObject properties, 
70
                          String partitions);
71

72
/**
73
 * Get table information by unique table ID
74
 * @param tableId Unique table identifier
75
 * @return TableInfo object or null if not found
76
 */
77
public TableInfo getTableInfoByTableId(String tableId);
78

79
/**
80
 * Get table information by table path
81
 * @param tablePath Full path to the table
82
 * @return TableInfo object or null if not found
83
 */
84
public TableInfo getTableInfoByPath(String tablePath);
85

86
/**
87
 * Get table information by table name in default namespace
88
 * @param tableName Short table name
89
 * @return TableInfo object or null if not found
90
 */
91
public TableInfo getTableInfoByName(String tableName);
92

93
/**
94
 * Get table information by table name and namespace
95
 * @param tableName Short table name
96
 * @param namespace Table namespace
97
 * @return TableInfo object or null if not found
98
 */
99
public TableInfo getTableInfoByNameAndNamespace(String tableName, String namespace);
100

101
/**
102
 * List all table paths in the system
103
 * @return List of all table paths
104
 */
105
public List<String> listTables();
106

107
/**
108
 * List table names in specific namespace
109
 * @param tableNamespace Target namespace
110
 * @return List of table names in the namespace
111
 */
112
public List<String> listTableNamesByNamespace(String tableNamespace);
113

114
/**
115
 * List table paths in specific namespace
116
 * @param tableNamespace Target namespace
117
 * @return List of table paths in the namespace
118
 */
119
public List<String> listTablePathsByNamespace(String tableNamespace);
120

121
/**
122
 * Get all table information objects in specific namespace
123
 * @param tableNamespace Target namespace
124
 * @return List of TableInfo objects
125
 */
126
public List<TableInfo> getTableInfosByNamespace(String tableNamespace);
127

128
/**
129
 * Update table schema for existing table
130
 * @param tableId Unique table identifier
131
 * @param tableSchema New JSON schema definition
132
 */
133
public void updateTableSchema(String tableId, String tableSchema);
134

135
/**
136
 * Update table properties
137
 * @param tableId Unique table identifier
138
 * @param properties New properties as JSON string
139
 */
140
public void updateTableProperties(String tableId, String properties);
141

142
/**
143
 * Update table short name mapping
144
 * @param tablePath Full table path
145
 * @param tableId Unique table identifier
146
 * @param tableName Short table name
147
 * @param tableNamespace Table namespace
148
 */
149
public void updateTableShortName(String tablePath, String tableId, String tableName, 
150
                                String tableNamespace);
151

152
/**
153
 * Delete table and all associated metadata
154
 * @param tablePath Full table path
155
 * @param tableId Unique table identifier
156
 * @param tableNamespace Table namespace
157
 */
158
public void deleteTableInfo(String tablePath, String tableId, String tableNamespace);
159

160
/**
161
 * Get table path from short table name
162
 * @param tableName Short table name
163
 * @param tableNamespace Table namespace
164
 * @return Full table path or null if not found
165
 */
166
public String getTablePathFromShortTableName(String tableName, String tableNamespace);
167
```
168

169
### Partition Operations
170

171
Comprehensive partition management including version control, snapshot handling, and incremental operations.
172

173
```java { .api }
174
/**
175
 * Get latest partition information for specific partition
176
 * @param tableId Unique table identifier
177
 * @param partitionDesc Partition description/key
178
 * @return Latest PartitionInfo or null if not found
179
 */
180
public PartitionInfo getSinglePartitionInfo(String tableId, String partitionDesc);
181

182
/**
183
 * Get partition information for specific version
184
 * @param tableId Unique table identifier
185
 * @param partitionDesc Partition description/key
186
 * @param version Specific partition version
187
 * @return PartitionInfo for the version or null if not found
188
 */
189
public PartitionInfo getSinglePartitionInfo(String tableId, String partitionDesc, int version);
190

191
/**
192
 * Get all partition information for a table
193
 * @param tableId Unique table identifier
194
 * @return List of all PartitionInfo objects for the table
195
 */
196
public List<PartitionInfo> getAllPartitionInfo(String tableId);
197

198
/**
199
 * Get all versions of a specific partition
200
 * @param tableId Unique table identifier
201
 * @param partitionDesc Partition description/key
202
 * @return List of PartitionInfo objects for all versions
203
 */
204
public List<PartitionInfo> getOnePartitionVersions(String tableId, String partitionDesc);
205

206
/**
207
 * Get all partition descriptions for a table
208
 * @param tableId Unique table identifier
209
 * @return List of partition description strings
210
 */
211
public List<String> getTableAllPartitionDesc(String tableId);
212

213
/**
214
 * Get latest timestamp for specific partition
215
 * @param tableId Unique table identifier
216
 * @param partitionDesc Partition description/key
217
 * @return Latest timestamp in milliseconds
218
 */
219
public long getLastedTimestamp(String tableId, String partitionDesc);
220

221
/**
222
 * Get latest version up to specific time
223
 * @param tableId Unique table identifier
224
 * @param partitionDesc Partition description/key
225
 * @param utcMills Timestamp in UTC milliseconds
226
 * @return Latest version number up to the specified time
227
 */
228
public int getLastedVersionUptoTime(String tableId, String partitionDesc, long utcMills);
229

230
/**
231
 * Get timestamp of latest version up to specific time
232
 * @param tableId Unique table identifier
233
 * @param partitionDesc Partition description/key
234
 * @param utcMills Timestamp in UTC milliseconds
235
 * @return Timestamp of latest version up to specified time
236
 */
237
public long getLastedVersionTimestampUptoTime(String tableId, String partitionDesc, long utcMills);
238

239
/**
240
 * Get incremental partitions between version range
241
 * @param tableId Unique table identifier
242
 * @param partitionDesc Partition description/key
243
 * @param startVersion Starting version (inclusive)
244
 * @param endVersion Ending version (inclusive)
245
 * @return List of PartitionInfo objects in version range
246
 */
247
public List<PartitionInfo> getIncrementalPartitions(String tableId, String partitionDesc, 
248
                                                   int startVersion, int endVersion);
249

250
/**
251
 * Get incremental partitions between timestamp range
252
 * @param tableId Unique table identifier
253
 * @param partitionDesc Partition description/key
254
 * @param startTimestamp Starting timestamp (inclusive)
255
 * @param endTimestamp Ending timestamp (inclusive)
256
 * @return List of PartitionInfo objects in timestamp range
257
 */
258
public List<PartitionInfo> getIncrementalPartitionsFromTimestamp(String tableId, String partitionDesc,
259
                                                                long startTimestamp, long endTimestamp);
260

261
/**
262
 * Delete all partition metadata for table
263
 * @param tableId Unique table identifier
264
 */
265
public void deletePartitionInfoByTableId(String tableId);
266

267
/**
268
 * Delete specific partition metadata
269
 * @param tableId Unique table identifier
270
 * @param partitionDesc Partition description/key
271
 */
272
public void deletePartitionInfoByTableAndPartition(String tableId, String partitionDesc);
273

274
/**
275
 * Logically delete all partitions (mark as deleted)
276
 * @param tableId Unique table identifier
277
 */
278
public void logicDeletePartitionInfoByTableId(String tableId);
279

280
/**
281
 * Logically delete specific partition (mark as deleted)
282
 * @param tableId Unique table identifier
283
 * @param partitionDesc Partition description/key
284
 */
285
public void logicDeletePartitionInfoByRangeId(String tableId, String partitionDesc);
286

287
/**
288
 * Rollback partition to specific version
289
 * @param tableId Unique table identifier
290
 * @param partitionDesc Partition description/key
291
 * @param version Target version to rollback to
292
 */
293
public void rollbackPartitionByVersion(String tableId, String partitionDesc, int version);
294
```
295

296
### Namespace Operations
297

298
Namespace management for organizing tables and implementing multi-tenancy.
299

300
```java { .api }
301
/**
302
 * Check if namespace exists
303
 * @param table_namespace Namespace name
304
 * @return true if namespace exists, false otherwise
305
 */
306
public boolean isNamespaceExists(String table_namespace);
307

308
/**
309
 * List all namespaces in the system
310
 * @return List of namespace names
311
 */
312
public List<String> listNamespaces();
313

314
/**
315
 * Create new namespace
316
 * @param name Namespace name
317
 * @param properties Namespace properties as JSON string
318
 * @param comment Optional comment (can be null)
319
 */
320
public void createNewNamespace(String name, String properties, String comment);
321

322
/**
323
 * Get namespace information by name
324
 * @param namespace Namespace name
325
 * @return Namespace object or null if not found
326
 */
327
public Namespace getNamespaceByNamespace(String namespace);
328

329
/**
330
 * Update namespace properties
331
 * @param namespace Namespace name
332
 * @param properties New properties as JSON string
333
 */
334
public void updateNamespaceProperties(String namespace, String properties);
335

336
/**
337
 * Delete namespace
338
 * @param namespace Namespace name to delete
339
 */
340
public void deleteNamespace(String namespace);
341
```
342

343
### Data Commit Operations
344

345
High-level data commit operations supporting ACID transactions and conflict resolution.
346

347
```java { .api }
348
/**
349
 * Commit data changes with comprehensive conflict resolution
350
 * @param metaInfo Metadata information containing partition and table info
351
 * @param changeSchema Whether schema changes are included
352
 * @param commitOp Type of commit operation
353
 * @return true if commit successful, false if conflicts occurred
354
 */
355
public boolean commitData(MetaInfo metaInfo, boolean changeSchema, CommitOp commitOp);
356

357
/**
358
 * Batch commit multiple data commit information
359
 * @param listData List of DataCommitInfo objects to commit
360
 * @return true if batch commit successful, false otherwise
361
 */
362
public boolean batchCommitDataCommitInfo(List<DataCommitInfo> listData);
363

364
/**
365
 * Commit single data commit with read partition info
366
 * @param dataCommitInfo Data commit information
367
 * @param readPartitionInfoList List of partitions read during operation
368
 */
369
public void commitDataCommitInfo(DataCommitInfo dataCommitInfo, 
370
                                List<PartitionInfo> readPartitionInfoList);
371

372
/**
373
 * Get all data file information for a partition
374
 * @param partitionInfo Partition information object
375
 * @return List of DataCommitInfo objects containing file operations
376
 */
377
public List<DataCommitInfo> getTableSinglePartitionDataInfo(PartitionInfo partitionInfo);
378

379
/**
380
 * Get partition snapshot for specific version
381
 * @param tableId Unique table identifier
382
 * @param partitionDesc Partition description/key
383
 * @param version Specific version number
384
 * @return List of DataCommitInfo objects for the snapshot
385
 */
386
public List<DataCommitInfo> getPartitionSnapshot(String tableId, String partitionDesc, int version);
387

388
/**
389
 * Get data commit information from UUIDs
390
 * @param tableId Unique table identifier
391
 * @param partitionDesc Partition description/key
392
 * @param dataCommitUUIDs List of commit UUIDs
393
 * @return List of DataCommitInfo objects
394
 */
395
public List<DataCommitInfo> getDataCommitInfosFromUUIDs(String tableId, String partitionDesc,
396
                                                        List<Uuid> dataCommitUUIDs);
397

398
/**
399
 * Delete data commit information
400
 * @param tableId Unique table identifier
401
 * @param partitionDesc Partition description/key
402
 * @param commitId Specific commit ID to delete
403
 */
404
public void deleteDataCommitInfo(String tableId, String partitionDesc, UUID commitId);
405

406
/**
407
 * Delete all data commit information for partition
408
 * @param tableId Unique table identifier
409
 * @param partitionDesc Partition description/key
410
 */
411
public void deleteDataCommitInfo(String tableId, String partitionDesc);
412

413
/**
414
 * Delete all data commit information for table
415
 * @param tableId Unique table identifier
416
 */
417
public void deleteDataCommitInfo(String tableId);
418
```
419

420
### Utility Operations
421

422
Utility operations for testing, cleanup, and metadata management.
423

424
```java { .api }
425
/**
426
 * Clean all metadata (for testing purposes only)
427
 * WARNING: This will delete all metadata in the system
428
 */
429
public void cleanMeta();
430

431
/**
432
 * Get file paths that should be deleted up to specific time
433
 * @param tableId Unique table identifier
434
 * @param partitionDesc Partition description/key (can be empty for all partitions)
435
 * @param utcMills Timestamp in UTC milliseconds
436
 * @return List of file paths that can be safely deleted
437
 */
438
public List<String> getDeleteFilePath(String tableId, String partitionDesc, long utcMills);
439

440
/**
441
 * Delete partition metadata and get associated file paths
442
 * @param tableId Unique table identifier
443
 * @param partitionDesc Partition description/key
444
 * @return List of file paths associated with deleted partition
445
 */
446
public List<String> deleteMetaPartitionInfo(String tableId, String partitionDesc);
447

448
/**
449
 * Logically drop columns from table
450
 * @param tableId Unique table identifier
451
 * @param droppedColumn List of column names to mark as dropped
452
 */
453
public void logicallyDropColumn(String tableId, List<String> droppedColumn);
454

455
/**
456
 * Remove logically dropped column information
457
 * @param tableId Unique table identifier
458
 */
459
public void removeLogicallyDropColumn(String tableId);
460
```
461

462
**Usage Examples:**
463

464
```java
465
import com.dmetasoul.lakesoul.meta.DBManager;
466
import com.dmetasoul.lakesoul.meta.entity.*;
467
import com.alibaba.fastjson.JSONObject;
468
import java.util.List;
469

470
// Initialize DBManager
471
DBManager dbManager = new DBManager();
472

473
// Create namespace
474
dbManager.createNewNamespace("analytics", "{\"owner\":\"data_team\"}", "Analytics namespace");
475

476
// Create table
477
JSONObject props = new JSONObject();
478
props.put("format", "parquet");
479
props.put("compression", "snappy");
480

481
dbManager.createNewTable(
482
    "tbl_001",
483
    "analytics", 
484
    "user_events",
485
    "/data/analytics/user_events",
486
    "{\"type\":\"struct\",\"fields\":[...]}",
487
    props,
488
    "date,hour"
489
);
490

491
// Check table existence
492
boolean exists = dbManager.isTableExists("/data/analytics/user_events");
493
if (exists) {
494
    // Get table info
495
    TableInfo tableInfo = dbManager.getTableInfoByPath("/data/analytics/user_events");
496
    System.out.println("Table schema: " + tableInfo.getTableSchema());
497
    
498
    // List partitions
499
    List<PartitionInfo> partitions = dbManager.getAllPartitionInfo("tbl_001");
500
    for (PartitionInfo partition : partitions) {
501
        System.out.println("Partition: " + partition.getPartitionDesc() + 
502
                          ", Version: " + partition.getVersion());
503
    }
504
}
505

506
// List tables in namespace
507
List<String> tables = dbManager.listTablePathsByNamespace("analytics");
508
System.out.println("Tables in analytics: " + tables);
509
```