0
# Compact Encoding
1

2
SCALE (Simple Concatenated Aggregate Little-Endian) compact encoding and decoding utilities for efficient data serialization used in Polkadot and Substrate blockchains.
3

4
## Capabilities
5

6
### Compact Encoding
7

8
Convert numbers to compact-encoded byte arrays for efficient storage and transmission.
9

10
```typescript { .api }
11
/**
12
 * Encodes value as compact Uint8Array
13
 * @param value - Value to encode (BN, bigint, number, or string)
14
 */
15
function compactToU8a(value: BN | bigint | number | string): Uint8Array;
16

17
/**
18
 * Adds compact encoded length prefix to data
19
 * @param input - Data to add length prefix to
20
 */
21
function compactAddLength(input: Uint8Array): Uint8Array;
22
```
23

24
### Compact Decoding
25

26
Decode compact-encoded byte arrays back to numeric values.
27

28
```typescript { .api }
29
/**
30
 * Decodes compact encoded value from Uint8Array
31
 * @param input - Compact encoded data
32
 * @returns Tuple of [decoded BN value, bytes consumed]
33
 */
34
function compactFromU8a(input: Uint8Array): [BN, number];
35

36
/**
37
 * Decodes compact encoded value with length limit
38
 * @param input - Compact encoded data
39
 * @param bitLength - Maximum bit length expected
40
 * @returns Tuple of [decoded BN value, bytes consumed]
41
 */
42
function compactFromU8aLim(input: Uint8Array, bitLength?: number): [BN, number];
43

44
/**
45
 * Removes compact encoded length prefix from data
46
 * @param input - Data with compact length prefix
47
 * @returns Tuple of [data without prefix, original length]
48
 */
49
function compactStripLength(input: Uint8Array): [Uint8Array, number];
50
```
51

52
## Usage Examples
53

54
**Basic Compact Encoding:**
55

56
```typescript
57
import { compactToU8a, compactFromU8a, BN } from "@polkadot/util";
58

59
// Encode small numbers (single-byte mode: 0-63)
60
const small = compactToU8a(42);
61
console.log(small); // Uint8Array(1) [168] (42 << 2 | 0b00)
62

63
// Encode medium numbers (two-byte mode: 64-16383)  
64
const medium = compactToU8a(1000);
65
console.log(medium); // Uint8Array(2) [161, 15] ((1000 << 2 | 0b01) in little-endian)
66

67
// Encode large numbers (four-byte mode: 16384-1073741823)
68
const large = compactToU8a(1000000);
69
console.log(large); // Uint8Array(4) [254, 36, 66, 15] ((1000000 << 2 | 0b10) in little-endian)
70

71
// Encode very large numbers (big-integer mode: >1073741823)
72
const veryLarge = compactToU8a(new BN("12345678901234567890"));
73
console.log(veryLarge); // Variable length with 0b11 prefix
74

75
// Decode back to original values
76
const [decodedSmall, bytesUsed1] = compactFromU8a(small);
77
console.log(decodedSmall.toNumber()); // 42
78
console.log(bytesUsed1); // 1
79

80
const [decodedMedium, bytesUsed2] = compactFromU8a(medium);
81
console.log(decodedMedium.toNumber()); // 1000
82
console.log(bytesUsed2); // 2
83
```
84

85
**Length-Prefixed Data:**
86

87
```typescript
88
import { compactAddLength, compactStripLength, stringToU8a, u8aToString } from "@polkadot/util";
89

90
// Add length prefix to data
91
const message = "Hello, Polkadot!";
92
const messageBytes = stringToU8a(message);
93
const withLength = compactAddLength(messageBytes);
94

95
console.log(`Original: ${messageBytes.length} bytes`); // 16 bytes
96
console.log(`With length prefix: ${withLength.length} bytes`); // 17 bytes (1 byte length + 16 bytes data)
97

98
// Remove length prefix and recover data
99
const [recovered, originalLength] = compactStripLength(withLength);
100
const recoveredMessage = u8aToString(recovered);
101

102
console.log(`Recovered length: ${originalLength}`); // 16
103
console.log(`Recovered message: ${recoveredMessage}`); // "Hello, Polkadot!"
104
console.log(`Match: ${recoveredMessage === message}`); // true
105
```
106

107
**Blockchain Data Serialization:**
108

109
```typescript
110
import { compactToU8a, compactFromU8a, u8aConcat } from "@polkadot/util";
111

112
// Serialize transaction data with compact encoding
113
interface Transaction {
114
  nonce: number;
115
  value: string;
116
  gasLimit: number;
117
  gasPrice: string;
118
}
119

120
function serializeTransaction(tx: Transaction): Uint8Array {
121
  return u8aConcat(
122
    compactToU8a(tx.nonce),          // Compact-encoded nonce
123
    compactToU8a(tx.value),          // Compact-encoded value
124
    compactToU8a(tx.gasLimit),       // Compact-encoded gas limit
125
    compactToU8a(tx.gasPrice)        // Compact-encoded gas price
126
  );
127
}
128

129
function deserializeTransaction(data: Uint8Array): Transaction {
130
  let offset = 0;
131
  
132
  const [nonce, nonceBytes] = compactFromU8a(data.slice(offset));
133
  offset += nonceBytes;
134
  
135
  const [value, valueBytes] = compactFromU8a(data.slice(offset));
136
  offset += valueBytes;
137
  
138
  const [gasLimit, gasLimitBytes] = compactFromU8a(data.slice(offset));
139
  offset += gasLimitBytes;
140
  
141
  const [gasPrice] = compactFromU8a(data.slice(offset));
142
  
143
  return {
144
    nonce: nonce.toNumber(),
145
    value: value.toString(),
146
    gasLimit: gasLimit.toNumber(),
147
    gasPrice: gasPrice.toString()
148
  };
149
}
150

151
// Example usage
152
const tx = {
153
  nonce: 42,
154
  value: "1000000000000000000", // 1 ETH in wei
155
  gasLimit: 21000,
156
  gasPrice: "20000000000" // 20 Gwei
157
};
158

159
const serialized = serializeTransaction(tx);
160
console.log(`Serialized size: ${serialized.length} bytes`);
161

162
const deserialized = deserializeTransaction(serialized);
163
console.log(deserialized);
164
// {
165
//   nonce: 42,
166
//   value: "1000000000000000000",
167
//   gasLimit: 21000, 
168
//   gasPrice: "20000000000"
169
// }
170
```
171

172
**Array Length Encoding:**
173

174
```typescript
175
import { compactToU8a, compactFromU8a, compactAddLength, u8aConcat } from "@polkadot/util";
176

177
// Encode array with compact length prefix
178
function encodeStringArray(strings: string[]): Uint8Array {
179
  // Encode array length
180
  const lengthBytes = compactToU8a(strings.length);
181
  
182
  // Encode each string with its length
183
  const encodedStrings = strings.map(str => {
184
    const strBytes = stringToU8a(str);
185
    return compactAddLength(strBytes);
186
  });
187
  
188
  return u8aConcat(lengthBytes, ...encodedStrings);
189
}
190

191
function decodeStringArray(data: Uint8Array): string[] {
192
  let offset = 0;
193
  
194
  // Decode array length
195
  const [length, lengthBytes] = compactFromU8a(data.slice(offset));
196
  offset += lengthBytes;
197
  
198
  const strings: string[] = [];
199
  const arrayLength = length.toNumber();
200
  
201
  // Decode each string
202
  for (let i = 0; i < arrayLength; i++) {
203
    const [strData, strLength] = compactStripLength(data.slice(offset));
204
    offset += compactToU8a(strLength).length + strLength;
205
    
206
    strings.push(u8aToString(strData));
207
  }
208
  
209
  return strings;
210
}
211

212
// Example
213
const originalArray = ["Alice", "Bob", "Charlie", "Diana"];
214
const encoded = encodeStringArray(originalArray);
215
const decoded = decodeStringArray(encoded);
216

217
console.log("Original:", originalArray);
218
console.log("Encoded size:", encoded.length, "bytes");
219
console.log("Decoded:", decoded);
220
console.log("Match:", JSON.stringify(originalArray) === JSON.stringify(decoded));
221
```
222

223
**Efficient Storage Optimization:**
224

225
```typescript
226
import { compactToU8a, numberToU8a } from "@polkadot/util";
227

228
// Compare compact vs fixed-size encoding
229
function compareEncodingEfficiency(values: number[]) {
230
  console.log("Value\t\tCompact\t\tFixed-32bit\t\tSavings");
231
  console.log("-----\t\t-------\t\t-----------\t\t-------");
232
  
233
  let totalCompactSize = 0;
234
  let totalFixedSize = 0;
235
  
236
  values.forEach(value => {
237
    const compactBytes = compactToU8a(value);
238
    const fixedBytes = numberToU8a(value, 32); // 32-bit fixed
239
    
240
    totalCompactSize += compactBytes.length;
241
    totalFixedSize += fixedBytes.length;
242
    
243
    const savings = ((fixedBytes.length - compactBytes.length) / fixedBytes.length * 100).toFixed(1);
244
    
245
    console.log(`${value}\t\t${compactBytes.length} bytes\t\t${fixedBytes.length} bytes\t\t${savings}%`);
246
  });
247
  
248
  const totalSavings = ((totalFixedSize - totalCompactSize) / totalFixedSize * 100).toFixed(1);
249
  console.log(`\nTotal: ${totalCompactSize} vs ${totalFixedSize} bytes (${totalSavings}% savings)`);
250
}
251

252
// Test with various value sizes
253
compareEncodingEfficiency([0, 1, 63, 64, 255, 16383, 16384, 65535, 1000000]);
254

255
// Output example:
256
// Value          Compact         Fixed-32bit     Savings
257
// -----          -------         -----------     -------
258
// 0              1 bytes         4 bytes         75.0%
259
// 1              1 bytes         4 bytes         75.0%
260
// 63             1 bytes         4 bytes         75.0%
261
// 64             2 bytes         4 bytes         50.0%
262
// 255            2 bytes         4 bytes         50.0%
263
// 16383          2 bytes         4 bytes         50.0%
264
// 16384          4 bytes         4 bytes         0.0%
265
// 65535          4 bytes         4 bytes         0.0%
266
// 1000000        4 bytes         4 bytes         0.0%
267
//
268
// Total: 21 vs 36 bytes (41.7% savings)
269
```
270

271
**Working with Limited Bit Lengths:**
272

273
```typescript
274
import { compactFromU8aLim, compactToU8a } from "@polkadot/util";
275

276
// Decode with bit length limits for validation
277
function safeCompactDecode(data: Uint8Array, maxBits: number): number | null {
278
  try {
279
    const [value, bytesUsed] = compactFromU8aLim(data, maxBits);
280
    
281
    // Additional validation
282
    if (value.bitLength() > maxBits) {
283
      console.warn(`Value exceeds ${maxBits} bits`);
284
      return null;
285
    }
286
    
287
    return value.toNumber();
288
  } catch (error) {
289
    console.error("Failed to decode compact value:", error);
290
    return null;
291
  }
292
}
293

294
// Test with various limits
295
const testValue = 65535; // 16-bit value
296
const encoded = compactToU8a(testValue);
297

298
console.log(safeCompactDecode(encoded, 16)); // 65535 (valid)
299
console.log(safeCompactDecode(encoded, 8));  // null (exceeds 8 bits)
300
console.log(safeCompactDecode(encoded, 32)); // 65535 (valid)
301
```
302

303
## Compact Encoding Specification
304

305
The SCALE compact encoding uses a variable-length format based on the value size:
306

307
- **Single-byte mode** (0-63): `value << 2 | 0b00`
308
- **Two-byte mode** (64-16383): `(value << 2 | 0b01)` in little-endian
309
- **Four-byte mode** (16384-1073741823): `(value << 2 | 0b10)` in little-endian  
310
- **Big-integer mode** (>1073741823): `(byte_length - 4) << 2 | 0b11` followed by value in little-endian
311

312
This encoding is essential for efficient data serialization in Substrate-based blockchains and ensures minimal storage overhead for common small values while supporting arbitrarily large numbers when needed.