Skip to content
Navigation Menu
{{ message }}
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathPersistentQueue.cpp
More file actions
538 lines (444 loc) · 14.3 KB
/
Copy pathPersistentQueue.cpp
File metadata and controls
538 lines (444 loc) · 14.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
#include <Arduino.h>
#include "PersistentQueue.h"
static const unsigned int pq_crc32tab[16] = {
0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190,
0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320, 0xf00f9344,
0xd6d6a3e8, 0xcb61b38c, 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278,
0xbdbdf21c
};
/**
* @brief calculates CRC32 value for a given buffer/length
*
* @param data - pointer to the array if usigned bytes
* @param length - length of the array
* @param crc previous value for incremental computation, 0xffffffff initially
* @return uint32_t - calculated CRC32
*/
uint32_t pq_crc32(const void *data, unsigned int length, uint32_t crc = 0xffffffff)
{
const unsigned char *buf = (const unsigned char *)data;
unsigned int i;
for (i = 0; i < length; ++i)
{
crc ^= buf[i];
crc = pq_crc32tab[crc & 0x0f] ^ (crc >> 4);
crc = pq_crc32tab[crc & 0x0f] ^ (crc >> 4);
}
// return value suitable for passing in next time, for final value invert it
return crc/* ^ 0xffffffff*/;
}
// namespace PQ {
/**
* @brief Construct a new Persistent Queue:: Persistent Queue object
*
* @param magic_num - message type ID
* @param dq_order - desired retrival order: latest message first or oldest message first
* @param calculate_crc - calculate and store CRC within the message file for consistency checking
*/
PersistentQueue::PersistentQueue(uint32_t magic_num, pqDequeueOrder_t dq_order, bool calculate_crc) {
m_magic = magic_num;
m_calcCRC = calculate_crc;
m_order = dq_order;
m_counter = 1;
m_initialized = false;
m_lastError = PQ_ERROR_OK;
}
/**
* @brief Destroy the Persistent Queue:: Persistent Queue object
*
*/
PersistentQueue::~PersistentQueue() {
}
/**
* @brief Initializes the PQ object and provides storage prefix
* prefix should start with an '/' and should not end with an '/'
*
* @param prefix
* @return true
* @return false
*/
bool PersistentQueue::begin(const char* prefix) {
m_prefix = prefix;
if ( m_prefix.length()>0 && !m_prefix.startsWith("/") ) m_prefix = "/" + m_prefix;
while ( m_prefix.endsWith("/") ) {
m_prefix.remove(m_prefix.length() - 1);
}
#ifdef PQ_USES_LITTLEFS
if ( !PQ_FS.exists(m_prefix) ) PQ_FS.mkdir(m_prefix);
#endif
m_initialized = true;
if ( !isQueueEmpty(false) ) {
// reset the internal counter to the latest number in case of restart after power failure
findNextMessage(false, PQ_DEQUEUE_LATEST);
m_counter = m_current_max+1;
// Serial.printf("PersistentQueue::begin - (%08x) m_counter set to %d (%d)\n", m_magic, m_counter, m_lastError);
}
m_lastError = PQ_ERROR_OK;
return true;
}
/**
* @brief Stop queue processing
*
*/
void PersistentQueue::end() {
m_lastError = PQ_ERROR_OK;
m_initialized = false;
}
bool PersistentQueue::checkInitialized() {
if ( !m_initialized) {
m_lastError = PQ_ERROR_NOT_INITIALIZED;
return false;
}
return true;
}
/**
* @brief Store message on the queue
*
* @param name - 32 bit number to be used to name the queue file. 0 if internal counter to be used
* developer to supply increasing numbers - the messages will be sorted by the name.
* if the same number is already used, a subnumber 0-99 will be used.
* messages with subnumbers are thought to have happened at the same time, therefore
* the concept of LATER/EARLIER does not apply
* messages are stored in: /<prefix>/1234567890-99 format
* @param data
* @param len
* @return true
* @return false
*/
bool PersistentQueue::enqueue(uint32_t name, const uint8_t* data, size_t len) {
if ( !checkInitialized()) return false;
// if zero is provided as a name number - use internal counter
if ( name == 0 ) name = m_counter++;
char fn[PQ_MAX_FILENAME_SIZE+1];
for (uint8_t i=0; i<PQ_MAX_SUBFILENAMES; i++) {
// 0123456789012345678
// '/q/1234567890-00
snprintf(fn, PQ_MAX_FILENAME_SIZE, "%s/%010d-%02d", m_prefix.c_str(), name, i);
// Serial.printf("PersistentQueue::enqueue - storing %s\n", fn);
if ( !PQ_FS.exists(fn) ) {
File F = PQ_FS.open(fn, "w+");
if ( !F ) {
m_lastError = PQ_ERROR_FILE_OP;
return false;
}
// write magic number first
F.write((const uint8_t*) &m_magic, sizeof(uint32_t));
// write message
F.write(data, len);
// write crc if requested
if ( m_calcCRC) {
uint32_t crc32 = pq_crc32(data, len);
F.write((const uint8_t*) &crc32, sizeof(uint32_t));
}
F.close();
m_lastError = PQ_ERROR_OK;
return true;
}
}
m_lastError = PQ_ERROR_OUT_OF_SUBNUMBERS;
return false;
}
/**
* @brief Check if queue is empty
*
* @param fast_check - assume all files are same correct type, do not check magic number
* @return true - the queue is empty
* @return false - queue has elements
*/
bool PersistentQueue::isQueueEmpty(bool fast_check) {
if ( !checkInitialized()) return false;
File root = PQ_FS.open(m_prefix);
if(!root || !root.isDirectory()) {
m_lastError = PQ_ERROR_INVALID_PREFIX;
return false;
}
m_lastError = PQ_ERROR_OK;
File file = root.openNextFile();
while (file) {
// for fast check we assume only messages of the same type (same magic number) are located in the queue folder
if ( fast_check ) {
file.close();
return false;
}
else {
uint32_t mn = 0;
file.read((uint8_t*) &mn, sizeof(uint32_t));
file.close();
if ( mn == m_magic ) return false;
}
file = root.openNextFile();
}
return true;
}
/**
* @brief Dequeue next message and place it into the provided buffer
* If buffer is too small, the actual length required could be checked after the call
*
* @param data - pointer to the buffer where the content of the message should be placed
* @param len - available length of the buffer
* @param actual_len - pointer to the variable to place actual length of the data read
* @param fast_check - assume all files are same correct type, do not check magic number
* @return true - message retrieved successfully
* @return false - message was not retrieved - check if buffer provided was large enough
*/
bool PersistentQueue::dequeue(uint8_t* data, size_t len, size_t* actual_len, bool fast_check) {
if ( !checkInitialized()) return false;
if ( isQueueEmpty() ) {
m_lastError = PQ_ERROR_QUEUE_EMPTY;
return false;
}
String fn = findNextMessage(fast_check);
String fp = fn;
// IDF 4.x and above returns just name, not a path like 3.x:
if ( !fn.startsWith("/")) fp = m_prefix+'/'+fn;
// Serial.printf("PersistentQueue::dequeue - retrieving %s (%s)\n", fp.c_str(), fn.c_str());
if ( fn.length() == 0 ) {
m_lastError = PQ_ERROR_QUEUE_EMPTY; // ?
return false;
}
File F = PQ_FS.open(fp, "r");
if ( !F ) {
m_lastError = PQ_ERROR_FILE_OP;
return false;
}
size_t expected_min_len = sizeof(uint32_t);
if ( m_calcCRC ) expected_min_len += sizeof(uint32_t);
size_t fsz = F.size();
if ( fsz < expected_min_len ) {
F.close();
m_lastError = PQ_ERROR_FILE_OP;
return false;
}
fsz -= expected_min_len;
// check magic number
uint32_t mn = 0;
F.read((uint8_t*) &mn, sizeof(uint32_t));
if ( mn != m_magic ) {
F.close();
m_lastError = PQ_ERROR_INVALID_MAGIC;
return false;
}
*actual_len = fsz;
if ( len < fsz ) {
m_lastError = PQ_ERROR_SMALL_BUFFER;
F.close();
return false;
}
if ( data == NULL ) {
m_lastError = PQ_ERROR_NULL_POINTER;
F.close();
return false;
}
memset(data, 0, len);
if ( F.read(data, fsz) != fsz ) {
m_lastError = PQ_ERROR_FILE_OP;
F.close();
return false;
}
if ( m_calcCRC ) {
uint32_t crc = 0;
if ( F.read((uint8_t*) &crc, sizeof(uint32_t)) != sizeof(uint32_t) || crc != pq_crc32(data, fsz) ) {
m_lastError = PQ_ERROR_BAD_CRC;
F.close();
return false;
}
}
F.close();
PQ_FS.remove(fp);
m_lastError = PQ_ERROR_OK;
return true;
}
/**
* @brief Dequeue next message, allocate memory for the contents
*
* @param data - pointer to a pointer - where the address of the buffer will be stored
* @param len - actual length of the dequeued message
* @param fast_check - assume all files are same correct type, do not check magic number
* @return true
* @return false
*/
bool PersistentQueue::dequeue(uint8_t** data, size_t* len, bool fast_check) {
if ( !checkInitialized()) return false;
if ( isQueueEmpty() ) {
m_lastError = PQ_ERROR_QUEUE_EMPTY;
return false;
}
String fn = findNextMessage(fast_check);
String fp = fn;
// IDF 4.x and above returns just name, not a path like 3.x:
if ( !fn.startsWith("/")) fp = m_prefix+'/'+fn;
// Serial.printf("PersistentQueue::dequeue - retrieving %s (%s)\n", fp.c_str(), fn.c_str());
if ( fn.length() == 0 ) {
m_lastError = PQ_ERROR_QUEUE_EMPTY; // ?
return false;
}
File F = PQ_FS.open(fp, "r");
if ( !F ) {
m_lastError = PQ_ERROR_FILE_OP;
return false;
}
size_t expected_min_len = sizeof(uint32_t);
if ( m_calcCRC ) expected_min_len += sizeof(uint32_t);
size_t fsz = F.size();
if ( fsz < expected_min_len ) {
F.close();
m_lastError = PQ_ERROR_FILE_OP;
return false;
}
fsz -= expected_min_len;
// check magic number
uint32_t mn = 0;
F.read((uint8_t*) &mn, sizeof(uint32_t));
if ( mn != m_magic ) {
F.close();
m_lastError = PQ_ERROR_INVALID_MAGIC;
return false;
}
uint8_t* p = (uint8_t*) malloc(fsz);
if ( p == NULL ) {
m_lastError = PQ_ERROR_OUT_OF_MEMORY;
F.close();
return false;
}
memset(p, 0, fsz);
if ( F.read(p, fsz) != fsz ) {
m_lastError = PQ_ERROR_FILE_OP;
F.close();
free(p);
p = NULL;
return false;
}
if ( m_calcCRC ) {
uint32_t crc = 0;
if ( F.read((uint8_t*) &crc, sizeof(uint32_t)) != sizeof(uint32_t) || crc != pq_crc32(p, fsz) ) {
m_lastError = PQ_ERROR_BAD_CRC;
F.close();
free(p);
p = NULL;
return false;
}
}
F.close();
PQ_FS.remove(fp);
*data = p;
*len = fsz;
m_lastError = PQ_ERROR_OK;
return true;
}
/**
* @brief Find next message in the queue based on the desired ordering
*
* @param fast_check - assume all files are same correct type, do not check magic number
* @return String - sought filename of the queued message
*/
String PersistentQueue::findNextMessage(bool fast_check, pqDequeueOrder_t order) {
if ( !checkInitialized()) return "";
pqDequeueOrder_t ord = (order == PQ_DEQUEUE_DEFAULT) ? m_order : order;
File root = PQ_FS.open(m_prefix);
if (!root || !root.isDirectory()) {
m_lastError = PQ_ERROR_INVALID_PREFIX;
return "";
}
File file = root.openNextFile();
uint32_t soughtNumber = (ord == PQ_DEQUEUE_OLDEST) ? UINT32_MAX : 1;
String soughtFileName = "";
while (file) {
if (file.isDirectory()) {
// Skip directories
file = root.openNextFile();
continue;
}
// if we are asked to check the message type - do it!
if ( !fast_check ) {
uint32_t mn = 0;
file.read((uint8_t*) &mn, sizeof(uint32_t));
if ( mn != m_magic ) {
file.close();
continue;
}
}
String fileName = file.name();
file.close(); // Close the current file before opening the next one
// Max number of digits in a 32 bit number is 10
// 0123456789012345678
// '/q/1234567890-00
// ' 1234567890
int prefixLen = m_prefix.length() + 1; // +1 for the '/'
String fn = fileName.substring(prefixLen, prefixLen + 10);
// Convert file name to a number
uint32_t fileNumber = fn.toInt();
// Check if the file name is a valid 32-bit number
if (fileNumber == 0 && fn != "0") {
file = root.openNextFile();
continue;
}
// Update the lowest number and file name if necessary
switch (ord) {
case PQ_DEQUEUE_OLDEST:
if (fileNumber < soughtNumber) {
soughtNumber = fileNumber;
soughtFileName = fileName;
}
break;
case PQ_DEQUEUE_LATEST:
if (fileNumber > soughtNumber) {
soughtNumber = fileNumber;
m_current_max = fileNumber;
soughtFileName = fileName;
}
break;
default:
break;
}
file = root.openNextFile();
}
return soughtFileName;
}
/**
* @brief Deletes all persistent messages from the queue folder
*
* @param fast_check - assume all files are same correct type, do not check magic number
* @return true - all messages deleted
* @return false - some messages were not deleted
*/
bool PersistentQueue::purge(bool fast_check) {
if ( !checkInitialized()) return false;
bool result = true;
File root = PQ_FS.open(m_prefix);
if (!root || !root.isDirectory()) {
m_lastError = PQ_ERROR_INVALID_PREFIX;
return false;
}
File file = root.openNextFile();
m_lastError = PQ_ERROR_OK;
while (file) {
if (file.isDirectory()) {
// Skip directories
file = root.openNextFile();
continue;
}
// if we are asked to check the message type - do it!
if ( !fast_check ) {
uint32_t mn = 0;
file.read((uint8_t*) &mn, sizeof(uint32_t));
if ( mn != m_magic ) {
file.close();
continue;
}
}
String fileName = file.name();
// IDF 4.x and above returns just name, not a path like 3.x:
if ( !fileName.startsWith("/") ) fileName = m_prefix + '/' + file.name();
file.close();
if ( !PQ_FS.remove(fileName) ) {
result = false;
m_lastError = PQ_ERROR_FILE_OP;
}
file = root.openNextFile();
}
#ifdef PQ_USES_LITTLEFS
PQ_FS.rmdir(m_prefix);
#endif
return result;
}
// }
You can’t perform that action at this time.