Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "elem.h"
21 : #include "import.h"
22 : #include "search.h"
23 : #include "state.h"
24 : #include "support.h"
25 : #include "parity.h"
26 : #include "stream.h"
27 : #include "handle.h"
28 : #include "io.h"
29 : #include "raid/raid.h"
30 : #include "raid/cpu.h"
31 :
32 : /**
33 : * Configure the multithread support.
34 : *
35 : * Multi thread for write could be either faster or slower, depending
36 : * on the specific conditions. With multithreads it's likely faster
37 : * writing to disk, but you'll need to access multiple times the same data,
38 : * being potentially slower.
39 : *
40 : * For upcoming SnapRAID version it's planned to add a mutex protection
41 : * at the file-system structure, slowing down multiple data access,
42 : * so we disable it.
43 : *
44 : * Multi thread for verify is instead always generally faster,
45 : * so we enable it if possible.
46 : */
47 : #if HAVE_PTHREAD
48 : /* #define HAVE_MT_WRITE 1 */
49 : #define HAVE_MT_VERIFY 1
50 : #endif
51 :
52 7 : const char* lev_name(unsigned l)
53 : {
54 7 : switch (l) {
55 2 : case 0 : return "Parity";
56 1 : case 1 : return "2-Parity";
57 1 : case 2 : return "3-Parity";
58 1 : case 3 : return "4-Parity";
59 1 : case 4 : return "5-Parity";
60 1 : case 5 : return "6-Parity";
61 : }
62 :
63 0 : return 0;
64 : }
65 :
66 186393 : const char* lev_config_name(unsigned l)
67 : {
68 186393 : switch (l) {
69 87377 : case 0 : return "parity";
70 59910 : case 1 : return "2-parity";
71 9816 : case 2 : return "3-parity";
72 9791 : case 3 : return "4-parity";
73 9766 : case 4 : return "5-parity";
74 9733 : case 5 : return "6-parity";
75 : }
76 :
77 0 : return 0;
78 : }
79 :
80 6345 : static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
81 : {
82 6345 : if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
83 449 : *level = 0;
84 449 : return 0;
85 : }
86 :
87 5896 : if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
88 225 : *level = 1;
89 225 : return 0;
90 : }
91 :
92 5671 : if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
93 200 : *level = 2;
94 200 : return 0;
95 : }
96 :
97 5471 : if (strcmp(s, "4-parity") == 0) {
98 197 : *level = 3;
99 197 : return 0;
100 : }
101 :
102 5274 : if (strcmp(s, "5-parity") == 0) {
103 194 : *level = 4;
104 194 : return 0;
105 : }
106 :
107 5080 : if (strcmp(s, "6-parity") == 0) {
108 190 : *level = 5;
109 190 : return 0;
110 : }
111 :
112 4890 : if (strcmp(s, "z-parity") == 0) {
113 0 : *level = 2;
114 0 : if (mode)
115 0 : *mode = RAID_MODE_VANDERMONDE;
116 0 : return 0;
117 : }
118 :
119 4890 : return -1;
120 : }
121 :
122 265 : const char* lev_raid_name(unsigned mode, unsigned n)
123 : {
124 265 : switch (n) {
125 41 : case 1 : return "par1";
126 25 : case 2 : return "par2";
127 3 : case 3 : if (mode == RAID_MODE_CAUCHY)
128 3 : return "par3";
129 : else
130 0 : return "parz";
131 3 : case 4 : return "par4";
132 4 : case 5 : return "par5";
133 189 : case 6 : return "par6";
134 : }
135 :
136 0 : return 0;
137 : }
138 :
139 267 : void state_init(struct snapraid_state* state)
140 : {
141 : unsigned l, s;
142 :
143 267 : memset(&state->opt, 0, sizeof(state->opt));
144 267 : state->filter_hidden = 0;
145 267 : state->autosave = 0;
146 267 : state->need_write = 0;
147 267 : state->checked_read = 0;
148 267 : state->block_size = 256 * KIBI; /* default 256 KiB */
149 267 : state->raid_mode = RAID_MODE_CAUCHY;
150 267 : state->file_mode = ADVISE_DEFAULT;
151 1869 : for (l = 0; l < LEV_MAX; ++l) {
152 1602 : state->parity[l].split_mac = 0;
153 14418 : for (s = 0; s < SPLIT_MAX; ++s) {
154 12816 : state->parity[l].split_map[s].path[0] = 0;
155 12816 : state->parity[l].split_map[s].uuid[0] = 0;
156 12816 : state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
157 12816 : state->parity[l].split_map[s].device = 0;
158 : }
159 1602 : state->parity[l].smartctl[0] = 0;
160 1602 : state->parity[l].total_blocks = 0;
161 1602 : state->parity[l].free_blocks = 0;
162 1602 : state->parity[l].skip_access = 0;
163 1602 : state->parity[l].tick = 0;
164 1602 : state->parity[l].cached = 0;
165 1602 : state->parity[l].is_excluded_by_filter = 0;
166 : }
167 267 : state->tick_io = 0;
168 267 : state->tick_misc = 0;
169 267 : state->tick_sched = 0;
170 267 : state->tick_raid = 0;
171 267 : state->tick_hash = 0;
172 267 : state->tick_last = tick();
173 267 : state->share[0] = 0;
174 267 : state->pool[0] = 0;
175 267 : state->pool_device = 0;
176 267 : state->lockfile[0] = 0;
177 267 : state->level = 1; /* default is the lowest protection */
178 267 : state->clear_past_hash = 0;
179 267 : state->no_conf = 0;
180 :
181 267 : tommy_list_init(&state->disklist);
182 267 : tommy_list_init(&state->maplist);
183 267 : tommy_list_init(&state->contentlist);
184 267 : tommy_list_init(&state->filterlist);
185 267 : tommy_list_init(&state->importlist);
186 267 : tommy_hashdyn_init(&state->importset);
187 267 : tommy_hashdyn_init(&state->previmportset);
188 267 : tommy_hashdyn_init(&state->searchset);
189 267 : tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
190 267 : }
191 :
192 244 : void state_done(struct snapraid_state* state)
193 : {
194 244 : tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
195 244 : tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
196 244 : tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
197 244 : tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
198 244 : tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
199 244 : tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
200 244 : tommy_hashdyn_done(&state->importset);
201 244 : tommy_hashdyn_done(&state->previmportset);
202 244 : tommy_hashdyn_done(&state->searchset);
203 244 : tommy_arrayblkof_done(&state->infoarr);
204 244 : }
205 :
206 : /**
207 : * Check the configuration.
208 : */
209 265 : static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
210 : {
211 : tommy_node* i;
212 : unsigned l, s;
213 :
214 : /* check for parity level */
215 265 : if (state->raid_mode == RAID_MODE_VANDERMONDE) {
216 0 : if (state->level > 3) {
217 : /* LCOV_EXCL_START */
218 : log_fatal("If you use the z-parity you cannot have more than 3 parities.\n");
219 : exit(EXIT_FAILURE);
220 : /* LCOV_EXCL_STIO */
221 : }
222 : }
223 :
224 : for (l = 0; l < state->level; ++l) {
225 : if (state->parity[l].split_mac == 0) {
226 : /* LCOV_EXCL_START */
227 : log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
228 : exit(EXIT_FAILURE);
229 : /* LCOV_EXCL_STOP */
230 : }
231 : }
232 :
233 265 : if (tommy_list_empty(&state->contentlist)) {
234 : /* LCOV_EXCL_START */
235 : log_fatal("No 'content' specification in '%s'\n", path);
236 : exit(EXIT_FAILURE);
237 : /* LCOV_EXCL_STOP */
238 : }
239 :
240 : /* check for equal paths */
241 1796 : for (i = state->contentlist; i != 0; i = i->next) {
242 1531 : struct snapraid_content* content = i->data;
243 :
244 9917 : for (l = 0; l < state->level; ++l) {
245 41930 : for (s = 0; s < state->parity[l].split_mac; ++s) {
246 33544 : if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
247 : /* LCOV_EXCL_START */
248 : log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
249 : exit(EXIT_FAILURE);
250 : /* LCOV_EXCL_STOP */
251 : }
252 : }
253 : }
254 : }
255 :
256 : /* check device of data disks */
257 265 : if (!state->opt.skip_device && !state->opt.skip_disk_access) {
258 0 : for (i = state->disklist; i != 0; i = i->next) {
259 : tommy_node* j;
260 0 : struct snapraid_disk* disk = i->data;
261 :
262 : /* skip data disks that are not accessible */
263 0 : if (disk->skip_access)
264 0 : continue;
265 :
266 : #ifdef _WIN32
267 : if (disk->device == 0) {
268 : /* LCOV_EXCL_START */
269 : log_fatal("Disk '%s' has a zero serial number.\n", disk->dir);
270 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
271 : log_fatal("it's better to change the serial number of the disk.\n");
272 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
273 : exit(EXIT_FAILURE);
274 : /* LCOV_EXCL_STOP */
275 : }
276 : #endif
277 :
278 0 : for (j = i->next; j != 0; j = j->next) {
279 0 : struct snapraid_disk* other = j->data;
280 0 : if (disk->device == other->device) {
281 0 : if (state->opt.force_device) {
282 : /* note tha we just ignore the issue */
283 : /* and we DON'T mark the disk to be skipped */
284 : /* because we want to use these disks */
285 0 : if (!state->opt.no_warnings)
286 0 : log_fatal("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
287 : } else {
288 : /* LCOV_EXCL_START */
289 : log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
290 : #ifdef _WIN32
291 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
292 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
293 : log_fatal("to change one of the disk serial.\n");
294 : #endif
295 : /* in "fix" we allow to continue anyway */
296 : if (strcmp(state->command, "fix") == 0) {
297 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
298 : }
299 : exit(EXIT_FAILURE);
300 : /* LCOV_EXCL_STOP */
301 : }
302 : }
303 : }
304 :
305 : /* skip data disks that are not accessible */
306 0 : if (disk->skip_access)
307 0 : continue;
308 :
309 0 : if (!state->opt.skip_parity_access) {
310 0 : for (l = 0; l < state->level; ++l) {
311 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
312 0 : if (disk->device == state->parity[l].split_map[s].device) {
313 : /* LCOV_EXCL_START */
314 : log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
315 : #ifdef _WIN32
316 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
317 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
318 : log_fatal("to change one of the disk serial.\n");
319 : #endif
320 : exit(EXIT_FAILURE);
321 : /* LCOV_EXCL_STOP */
322 : }
323 : }
324 : }
325 : }
326 :
327 0 : if (state->pool[0] != 0 && disk->device == state->pool_device) {
328 : /* LCOV_EXCL_START */
329 : log_fatal("Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
330 : #ifdef _WIN32
331 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
332 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
333 : log_fatal("to change one of the disk serial.\n");
334 : #endif
335 : exit(EXIT_FAILURE);
336 : /* LCOV_EXCL_STOP */
337 : }
338 : }
339 : }
340 :
341 : /* check device of parity disks */
342 265 : if (!state->opt.skip_device && !state->opt.skip_parity_access) {
343 0 : for (l = 0; l < state->level; ++l) {
344 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
345 : unsigned j, t;
346 :
347 : /* skip parity disks that are not accessible */
348 0 : if (state->parity[l].skip_access)
349 0 : continue;
350 :
351 : #ifdef _WIN32
352 : if (state->parity[l].split_map[s].device == 0) {
353 : /* LCOV_EXCL_START */
354 : log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
355 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
356 : log_fatal("it's better to change the serial number of the disk.\n");
357 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
358 : exit(EXIT_FAILURE);
359 : /* LCOV_EXCL_STOP */
360 : }
361 : #endif
362 :
363 0 : for (j = l + 1; j < state->level; ++j) {
364 0 : for (t = 0; t < state->parity[j].split_mac; ++t) {
365 0 : if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
366 0 : if (state->opt.force_device) {
367 : /* note tha we just ignore the issue */
368 : /* and we DON'T mark the disk to be skipped */
369 : /* because we want to use these disks */
370 0 : if (!state->opt.no_warnings)
371 0 : log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
372 : } else {
373 : /* LCOV_EXCL_START */
374 : log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
375 : #ifdef _WIN32
376 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
377 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
378 : log_fatal("to change one of the disk serial.\n");
379 : #endif
380 : /* in "fix" we allow to continue anyway */
381 : if (strcmp(state->command, "fix") == 0) {
382 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
383 : }
384 : exit(EXIT_FAILURE);
385 : /* LCOV_EXCL_STOP */
386 : }
387 : }
388 : }
389 : }
390 :
391 0 : if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
392 : /* LCOV_EXCL_START */
393 : log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
394 : #ifdef _WIN32
395 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
396 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
397 : log_fatal("to change one of the disk serial.\n");
398 : #endif
399 : exit(EXIT_FAILURE);
400 : /* LCOV_EXCL_STOP */
401 : }
402 : }
403 : }
404 : }
405 :
406 : /* check device of pool disk */
407 : #ifdef _WIN32
408 : if (!state->opt.skip_device) {
409 : if (state->pool[0] != 0 && state->pool_device == 0) {
410 : /* LCOV_EXCL_START */
411 : log_fatal("Disk '%s' has a zero serial number.\n", state->pool);
412 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
413 : log_fatal("it's better to change the serial number of the disk.\n");
414 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
415 : exit(EXIT_FAILURE);
416 : /* LCOV_EXCL_STOP */
417 : }
418 : }
419 : #endif
420 :
421 : /* count the content files */
422 265 : if (!state->opt.skip_device && !state->opt.skip_content_access) {
423 : unsigned content_count;
424 :
425 0 : content_count = 0;
426 0 : for (i = state->contentlist; i != 0; i = i->next) {
427 : tommy_node* j;
428 0 : struct snapraid_content* content = i->data;
429 :
430 : /* check if there are others in the same disk */
431 0 : for (j = i->next; j != 0; j = j->next) {
432 0 : struct snapraid_content* other = j->data;
433 0 : if (content->device == other->device) {
434 0 : log_fatal("WARNING! Content files on the same disk: '%s' and '%s'.\n", content->content, other->content);
435 0 : break;
436 : }
437 : }
438 0 : if (j != 0) {
439 : /* skip it */
440 0 : continue;
441 : }
442 :
443 0 : ++content_count;
444 : }
445 :
446 0 : if (content_count < state->level + 1) {
447 : /* LCOV_EXCL_START */
448 : log_fatal("You must have at least %d 'content' files in different disks.\n", state->level + 1);
449 : exit(EXIT_FAILURE);
450 : /* LCOV_EXCL_STOP */
451 : }
452 : }
453 :
454 : /* check for speed */
455 : #ifdef CONFIG_X86
456 265 : if (!raid_cpu_has_ssse3())
457 : #endif
458 0 : if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
459 0 : if (state->level == 3) {
460 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
461 0 : log_fatal("WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
462 0 : } else if (state->level > 3) {
463 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
464 0 : log_fatal("WARNING! It's recommended to reduce the parity levels to triple parity.\n");
465 : }
466 : }
467 :
468 : /* ensure that specified filter disks are valid ones */
469 275 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
470 : tommy_node* j;
471 10 : struct snapraid_filter* filter = i->data;
472 51 : for (j = state->disklist; j != 0; j = j->next) {
473 45 : struct snapraid_disk* disk = j->data;
474 45 : if (fnmatch(filter->pattern, disk->name, FNM_CASEINSENSITIVE_FOR_WIN) == 0)
475 4 : break;
476 : }
477 10 : if (j == 0) {
478 : /* check matching with parity disks */
479 21 : for (l = 0; l < state->level; ++l)
480 21 : if (fnmatch(filter->pattern, lev_config_name(l), FNM_CASEINSENSITIVE_FOR_WIN) == 0)
481 6 : break;
482 6 : if (l == state->level) {
483 : /* LCOV_EXCL_START */
484 : log_fatal("Option -d, --filter-disk %s doesn't match any data or parity disk.\n", filter->pattern);
485 : exit(EXIT_FAILURE);
486 : /* LCOV_EXCL_STOP */
487 : }
488 : }
489 : }
490 265 : }
491 :
492 : /**
493 : * Validate the smartctl command.
494 : *
495 : * It must contains only one %s string, and not other % chars.
496 : */
497 366 : static int validate_smartctl(const char* custom)
498 : {
499 366 : const char* s = custom;
500 366 : int arg = 0;
501 :
502 2562 : while (*s) {
503 1830 : if (s[0] == '%' && s[1] == 's') {
504 183 : if (arg) {
505 : /* LCOV_EXCL_START */
506 : return -1;
507 : /* LCOV_EXCL_STOP */
508 : }
509 183 : arg = 1;
510 1647 : } else if (s[0] == '%') {
511 : /* LCOV_EXCL_START */
512 : return -1;
513 : /* LCOV_EXCL_STOP */
514 : }
515 :
516 1830 : ++s;
517 : }
518 :
519 366 : return 0;
520 : }
521 :
522 266 : void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
523 : {
524 : STREAM* f;
525 : unsigned line;
526 : tommy_node* i;
527 : unsigned l, s;
528 :
529 : /* copy the options */
530 266 : state->opt = *opt;
531 :
532 : /* if unset, sort by physical order */
533 266 : if (!state->opt.force_order)
534 1 : state->opt.force_order = SORT_PHYSICAL;
535 :
536 : /* adjust file mode */
537 266 : if (state->opt.file_mode != ADVISE_DEFAULT) {
538 4 : state->file_mode = state->opt.file_mode;
539 : } else {
540 : /* default mode, if nothing is specified */
541 262 : state->file_mode = ADVISE_FLUSH;
542 : }
543 :
544 : /* store current command */
545 266 : state->command = command;
546 :
547 266 : log_tag("conf:file:%s\n", path);
548 :
549 266 : f = sopen_read(path);
550 266 : if (!f) {
551 : /* LCOV_EXCL_START */
552 : if (errno == ENOENT) {
553 : if (state->opt.auto_conf) {
554 : log_tag("conf:missing:\n");
555 : /* mark that we are without a configuration file */
556 : state->no_conf = 1;
557 : state->level = 0;
558 : return;
559 : }
560 :
561 : log_fatal("No configuration file found at '%s'\n", path);
562 : } else if (errno == EACCES) {
563 : log_fatal("You do not have rights to access the configuration file '%s'\n", path);
564 : } else {
565 : log_fatal("Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
566 : }
567 : exit(EXIT_FAILURE);
568 : /* LCOV_EXCL_STOP */
569 : }
570 :
571 266 : line = 1;
572 : while (1) {
573 : char tag[PATH_MAX];
574 : char buffer[PATH_MAX];
575 : int ret;
576 : int c;
577 : unsigned level;
578 :
579 : /* skip initial spaces */
580 6245 : sgetspace(f);
581 :
582 : /* read the command */
583 6245 : ret = sgettok(f, tag, sizeof(tag));
584 6245 : if (ret < 0) {
585 : /* LCOV_EXCL_START */
586 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
587 : exit(EXIT_FAILURE);
588 : /* LCOV_EXCL_STOP */
589 : }
590 :
591 : /* skip spaces after the command */
592 6245 : sgetspace(f);
593 :
594 6245 : if (strcmp(tag, "blocksize") == 0
595 : /* block_size is the old format of the option */
596 5979 : || strcmp(tag, "block_size") == 0) {
597 :
598 266 : ret = sgetu32(f, &state->block_size);
599 266 : if (ret < 0) {
600 : /* LCOV_EXCL_START */
601 : log_fatal("Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
602 : exit(EXIT_FAILURE);
603 : /* LCOV_EXCL_STOP */
604 : }
605 266 : if (state->block_size < 1) {
606 : /* LCOV_EXCL_START */
607 : log_fatal("Too small 'blocksize' specification in '%s' at line %u\n", path, line);
608 : exit(EXIT_FAILURE);
609 : /* LCOV_EXCL_STOP */
610 : }
611 266 : if (state->block_size > 16 * KIBI) {
612 : /* LCOV_EXCL_START */
613 : log_fatal("Too big 'blocksize' specification in '%s' at line %u\n", path, line);
614 : exit(EXIT_FAILURE);
615 : /* LCOV_EXCL_STOP */
616 : }
617 : /* check if it's a power of 2 */
618 266 : if ((state->block_size & (state->block_size - 1)) != 0) {
619 : /* LCOV_EXCL_START */
620 : log_fatal("Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
621 : exit(EXIT_FAILURE);
622 : /* LCOV_EXCL_STOP */
623 : }
624 266 : state->block_size *= KIBI;
625 5979 : } else if (strcmp(tag, "hashsize") == 0
626 5979 : || strcmp(tag, "hash_size") == 0 /* v11.0 used incorretly this one, kept now for backward compatibility */
627 0 : ) {
628 : uint32_t hash_size;
629 :
630 0 : ret = sgetu32(f, &hash_size);
631 0 : if (ret < 0) {
632 : /* LCOV_EXCL_START */
633 : log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
634 : exit(EXIT_FAILURE);
635 : /* LCOV_EXCL_STOP */
636 : }
637 0 : if (hash_size < 2) {
638 : /* LCOV_EXCL_START */
639 : log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
640 : exit(EXIT_FAILURE);
641 : /* LCOV_EXCL_STOP */
642 : }
643 0 : if (hash_size > HASH_MAX) {
644 : /* LCOV_EXCL_START */
645 : log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
646 : exit(EXIT_FAILURE);
647 : /* LCOV_EXCL_STOP */
648 : }
649 : /* check if it's a power of 2 */
650 0 : if ((hash_size & (hash_size - 1)) != 0) {
651 : /* LCOV_EXCL_START */
652 : log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
653 : exit(EXIT_FAILURE);
654 : /* LCOV_EXCL_STOP */
655 : }
656 :
657 0 : BLOCK_HASH_SIZE = hash_size;
658 5979 : } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
659 : char device[PATH_MAX];
660 : char* split_map[SPLIT_MAX + 1];
661 : unsigned split_mac;
662 : char* slash;
663 : uint64_t dev;
664 : int skip_access;
665 :
666 1272 : if (state->parity[level].split_mac != 0) {
667 : /* LCOV_EXCL_START */
668 : log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
669 : exit(EXIT_FAILURE);
670 : /* LCOV_EXCL_STOP */
671 : }
672 :
673 1272 : ret = sgetlasttok(f, buffer, sizeof(buffer));
674 1272 : if (ret < 0) {
675 : /* LCOV_EXCL_START */
676 : log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
677 : exit(EXIT_FAILURE);
678 : /* LCOV_EXCL_STOP */
679 : }
680 :
681 1272 : if (!*buffer) {
682 : /* LCOV_EXCL_START */
683 : log_fatal("Empty '%s' specification in '%s' at line %u\n", tag, path, line);
684 : exit(EXIT_FAILURE);
685 : /* LCOV_EXCL_STOP */
686 : }
687 :
688 1272 : split_mac = strsplit(split_map, SPLIT_MAX + 1, buffer, ",");
689 :
690 1272 : if (split_mac > SPLIT_MAX) {
691 : /* LCOV_EXCL_START */
692 : log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
693 : exit(EXIT_FAILURE);
694 : /* LCOV_EXCL_STOP */
695 : }
696 :
697 1272 : skip_access = 0;
698 1272 : state->parity[level].split_mac = split_mac;
699 6356 : for (s = 0; s < split_mac; ++s) {
700 5085 : pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
701 :
702 5085 : if (!state->opt.skip_parity_access) {
703 : struct stat st;
704 :
705 : /* get the device of the directory containing the parity file */
706 4349 : pathimport(device, sizeof(device), split_map[s]);
707 4349 : slash = strrchr(device, '/');
708 4349 : if (slash)
709 4349 : *slash = 0;
710 : else
711 0 : pathcpy(device, sizeof(device), ".");
712 :
713 4349 : if (stat(device, &st) == 0) {
714 4344 : dev = st.st_dev;
715 : } else {
716 : /* if the disk can be skipped */
717 5 : if (state->opt.force_device) {
718 : /* use a fake device, and mark the disk to be skipped */
719 4 : dev = 0;
720 4 : skip_access = 1;
721 4 : log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
722 : } else {
723 : /* LCOV_EXCL_START */
724 : log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
725 :
726 : /* in "fix" we allow to continue anyway */
727 : if (strcmp(state->command, "fix") == 0) {
728 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
729 : }
730 : exit(EXIT_FAILURE);
731 : /* LCOV_EXCL_STOP */
732 : }
733 : }
734 : } else {
735 : /* use a fake device */
736 736 : dev = 0;
737 : }
738 :
739 5084 : state->parity[level].split_map[s].device = dev;
740 : }
741 :
742 : /* store the global parity skip_access */
743 1271 : state->parity[level].skip_access = skip_access;
744 :
745 : /* adjust the level */
746 1271 : if (state->level < level + 1)
747 1005 : state->level = level + 1;
748 4707 : } else if (strcmp(tag, "share") == 0) {
749 41 : if (*state->share) {
750 : /* LCOV_EXCL_START */
751 : log_fatal("Multiple 'share' specification in '%s' at line %u\n", path, line);
752 : exit(EXIT_FAILURE);
753 : /* LCOV_EXCL_STOP */
754 : }
755 :
756 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
757 41 : if (ret < 0) {
758 : /* LCOV_EXCL_START */
759 : log_fatal("Invalid 'share' specification in '%s' at line %u\n", path, line);
760 : exit(EXIT_FAILURE);
761 : /* LCOV_EXCL_STOP */
762 : }
763 :
764 41 : if (!*buffer) {
765 : /* LCOV_EXCL_START */
766 : log_fatal("Empty 'share' specification in '%s' at line %u\n", path, line);
767 : exit(EXIT_FAILURE);
768 : /* LCOV_EXCL_STOP */
769 : }
770 :
771 41 : pathimport(state->share, sizeof(state->share), buffer);
772 4666 : } else if (strcmp(tag, "pool") == 0) {
773 : struct stat st;
774 :
775 41 : if (*state->pool) {
776 : /* LCOV_EXCL_START */
777 : log_fatal("Multiple 'pool' specification in '%s' at line %u\n", path, line);
778 : exit(EXIT_FAILURE);
779 : /* LCOV_EXCL_STOP */
780 : }
781 :
782 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
783 41 : if (ret < 0) {
784 : /* LCOV_EXCL_START */
785 : log_fatal("Invalid 'pool' specification in '%s' at line %u\n", path, line);
786 : exit(EXIT_FAILURE);
787 : /* LCOV_EXCL_STOP */
788 : }
789 :
790 41 : if (!*buffer) {
791 : /* LCOV_EXCL_START */
792 : log_fatal("Empty 'pool' specification in '%s' at line %u\n", path, line);
793 : exit(EXIT_FAILURE);
794 : /* LCOV_EXCL_STOP */
795 : }
796 :
797 41 : pathimport(state->pool, sizeof(state->pool), buffer);
798 :
799 : /* get the device of the directory containing the pool tree */
800 41 : if (stat(buffer, &st) != 0) {
801 : /* LCOV_EXCL_START */
802 : log_fatal("Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
803 : exit(EXIT_FAILURE);
804 : /* LCOV_EXCL_STOP */
805 : }
806 :
807 41 : state->pool_device = st.st_dev;
808 4625 : } else if (strcmp(tag, "content") == 0) {
809 : struct snapraid_content* content;
810 : char device[PATH_MAX];
811 : char* slash;
812 : struct stat st;
813 : uint64_t dev;
814 :
815 1531 : ret = sgetlasttok(f, buffer, sizeof(buffer));
816 1531 : if (ret < 0) {
817 : /* LCOV_EXCL_START */
818 : log_fatal("Invalid 'content' specification in '%s' at line %u\n", path, line);
819 : exit(EXIT_FAILURE);
820 : /* LCOV_EXCL_STOP */
821 : }
822 :
823 1531 : if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
824 : /* LCOV_EXCL_START */
825 : log_fatal("You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
826 : exit(EXIT_FAILURE);
827 : /* LCOV_EXCL_STOP */
828 : }
829 :
830 1531 : if (!*buffer) {
831 : /* LCOV_EXCL_START */
832 : log_fatal("Empty 'content' specification in '%s' at line %u\n", path, line);
833 : exit(EXIT_FAILURE);
834 : /* LCOV_EXCL_STOP */
835 : }
836 :
837 : /* check if the content file is already specified */
838 5724 : for (i = state->contentlist; i != 0; i = i->next) {
839 4193 : content = i->data;
840 4193 : if (pathcmp(content->content, buffer) == 0)
841 0 : break;
842 : }
843 1531 : if (i) {
844 : /* LCOV_EXCL_START */
845 : log_fatal("Duplicate 'content' specification in '%s' at line %u\n", path, line);
846 : exit(EXIT_FAILURE);
847 : /* LCOV_EXCL_STOP */
848 : }
849 :
850 : /* get the device of the directory containing the content file */
851 1531 : pathimport(device, sizeof(device), buffer);
852 1531 : slash = strrchr(device, '/');
853 1531 : if (slash)
854 1531 : *slash = 0;
855 : else
856 0 : pathcpy(device, sizeof(device), ".");
857 1531 : if (stat(device, &st) == 0) {
858 1530 : dev = st.st_dev;
859 : } else {
860 1 : if (state->opt.skip_content_access) {
861 : /* use a fake device */
862 1 : dev = 0;
863 1 : log_fatal("WARNING! Skipping inaccessible content file '%s'...\n", buffer);
864 : } else {
865 : /* LCOV_EXCL_START */
866 : log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
867 : exit(EXIT_FAILURE);
868 : /* LCOV_EXCL_STOP */
869 : }
870 : }
871 :
872 : /* set the lock file at the first accessible content file */
873 1531 : if (state->lockfile[0] == 0 && dev != 0) {
874 265 : pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
875 265 : pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
876 : }
877 :
878 1531 : content = content_alloc(buffer, dev);
879 :
880 1531 : tommy_list_insert_tail(&state->contentlist, &content->node, content);
881 4679 : } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
882 : /* "disk" is the deprecated name up to SnapRAID 9.x */
883 : char dir[PATH_MAX];
884 : char device[PATH_MAX];
885 : char uuid[UUID_MAX];
886 : struct snapraid_disk* disk;
887 : uint64_t dev;
888 : int skip_access;
889 :
890 1585 : ret = sgettok(f, buffer, sizeof(buffer));
891 1585 : if (ret < 0) {
892 : /* LCOV_EXCL_START */
893 : log_fatal("Invalid 'data' name specification in '%s' at line %u\n", path, line);
894 : exit(EXIT_FAILURE);
895 : /* LCOV_EXCL_STOP */
896 : }
897 :
898 1585 : if (!*buffer) {
899 : /* LCOV_EXCL_START */
900 : log_fatal("Empty 'data' name specification in '%s' at line %u\n", path, line);
901 : exit(EXIT_FAILURE);
902 : /* LCOV_EXCL_STOP */
903 : }
904 :
905 1585 : sgetspace(f);
906 :
907 1585 : ret = sgetlasttok(f, dir, sizeof(dir));
908 1585 : if (ret < 0) {
909 : /* LCOV_EXCL_START */
910 : log_fatal("Invalid 'data' dir specification in '%s' at line %u\n", path, line);
911 : exit(EXIT_FAILURE);
912 : /* LCOV_EXCL_STOP */
913 : }
914 :
915 1585 : if (!*dir) {
916 : /* LCOV_EXCL_START */
917 : log_fatal("Empty 'data' dir specification in '%s' at line %u\n", path, line);
918 : exit(EXIT_FAILURE);
919 : /* LCOV_EXCL_STOP */
920 : }
921 :
922 : /* get the device of the dir */
923 1585 : pathimport(device, sizeof(device), dir);
924 :
925 : /* check if the disk name already exists */
926 5535 : for (i = state->disklist; i != 0; i = i->next) {
927 3950 : disk = i->data;
928 3950 : if (strcmp(disk->name, buffer) == 0)
929 0 : break;
930 : }
931 1585 : if (i) {
932 : /* LCOV_EXCL_START */
933 : log_fatal("Duplicate 'data' name '%s' at line %u\n", buffer, line);
934 : exit(EXIT_FAILURE);
935 : /* LCOV_EXCL_STOP */
936 : }
937 :
938 : /* if the disk has to be present */
939 1585 : skip_access = 0;
940 1585 : if (!state->opt.skip_disk_access) {
941 : struct stat st;
942 :
943 1423 : if (stat(device, &st) == 0) {
944 1422 : dev = st.st_dev;
945 :
946 : /* read the uuid, if unsupported use an empty one */
947 1422 : if (devuuid(dev, uuid, sizeof(uuid)) != 0) {
948 0 : *uuid = 0;
949 : }
950 :
951 : /* fake a different UUID when testing */
952 1422 : if (state->opt.fake_uuid) {
953 2 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
954 2 : --state->opt.fake_uuid;
955 : }
956 : } else {
957 : /* if the disk can be skipped */
958 1 : if (state->opt.force_device) {
959 : /* use a fake device, and mark the disk to be skipped */
960 1 : dev = 0;
961 1 : *uuid = 0;
962 1 : skip_access = 1;
963 1 : log_fatal("DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
964 : } else {
965 : /* LCOV_EXCL_START */
966 : log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
967 :
968 : /* in "fix" we allow to continue anyway */
969 : if (strcmp(state->command, "fix") == 0) {
970 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
971 : }
972 : exit(EXIT_FAILURE);
973 : /* LCOV_EXCL_STOP */
974 : }
975 : }
976 : } else {
977 : /* use a fake device */
978 162 : dev = 0;
979 162 : *uuid = 0;
980 : }
981 :
982 1585 : disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
983 :
984 1585 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
985 1509 : } else if (strcmp(tag, "smartctl") == 0) {
986 : char custom[PATH_MAX];
987 :
988 366 : ret = sgettok(f, buffer, sizeof(buffer));
989 366 : if (ret < 0) {
990 : /* LCOV_EXCL_START */
991 : log_fatal("Invalid 'smartctl' name specification in '%s' at line %u\n", path, line);
992 : exit(EXIT_FAILURE);
993 : /* LCOV_EXCL_STOP */
994 : }
995 :
996 366 : if (!*buffer) {
997 : /* LCOV_EXCL_START */
998 : log_fatal("Empty 'smartctl' name specification in '%s' at line %u\n", path, line);
999 : exit(EXIT_FAILURE);
1000 : /* LCOV_EXCL_STOP */
1001 : }
1002 :
1003 366 : sgetspace(f);
1004 :
1005 366 : ret = sgetlasttok(f, custom, sizeof(custom));
1006 366 : if (ret < 0) {
1007 : /* LCOV_EXCL_START */
1008 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1009 : exit(EXIT_FAILURE);
1010 : /* LCOV_EXCL_STOP */
1011 : }
1012 :
1013 366 : if (!*custom) {
1014 : /* LCOV_EXCL_START */
1015 : log_fatal("Empty 'smartctl' option specification in '%s' at line %u\n", path, line);
1016 : exit(EXIT_FAILURE);
1017 : /* LCOV_EXCL_STOP */
1018 : }
1019 :
1020 366 : if (validate_smartctl(custom) != 0) {
1021 : /* LCOV_EXCL_START */
1022 : log_fatal("Invalid 'smartctl' option specification in '%s' at line %u\n", path, line);
1023 : exit(EXIT_FAILURE);
1024 : /* LCOV_EXCL_STOP */
1025 : }
1026 :
1027 : /* search for parity */
1028 366 : if (lev_config_scan(buffer, &level, 0) == 0) {
1029 183 : if (state->parity[level].smartctl[0] != 0) {
1030 : /* LCOV_EXCL_START */
1031 : log_fatal("Duplicate parity smartctl '%s' at line %u\n", buffer, line);
1032 : exit(EXIT_FAILURE);
1033 : /* LCOV_EXCL_STOP */
1034 : }
1035 :
1036 183 : pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1037 : } else {
1038 : struct snapraid_disk* disk;
1039 :
1040 : /* search the disk */
1041 183 : disk = 0;
1042 183 : for (i = state->disklist; i != 0; i = i->next) {
1043 183 : disk = i->data;
1044 183 : if (strcmp(disk->name, buffer) == 0)
1045 183 : break;
1046 : }
1047 183 : if (!disk) {
1048 : /* LCOV_EXCL_START */
1049 : log_fatal("Missing disk smartctl '%s' at line %u\n", buffer, line);
1050 : exit(EXIT_FAILURE);
1051 : /* LCOV_EXCL_STOP */
1052 : }
1053 183 : if (disk->smartctl[0] != 0) {
1054 : /* LCOV_EXCL_START */
1055 : log_fatal("Duplicate disk name '%s' at line %u\n", buffer, line);
1056 : exit(EXIT_FAILURE);
1057 : /* LCOV_EXCL_STOP */
1058 : }
1059 :
1060 183 : pathcpy(disk->smartctl, sizeof(disk->smartctl), custom);
1061 : }
1062 1143 : } else if (strcmp(tag, "nohidden") == 0) {
1063 0 : state->filter_hidden = 1;
1064 1143 : } else if (strcmp(tag, "exclude") == 0) {
1065 : struct snapraid_filter* filter;
1066 :
1067 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1068 265 : if (ret < 0) {
1069 : /* LCOV_EXCL_START */
1070 : log_fatal("Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1071 : exit(EXIT_FAILURE);
1072 : /* LCOV_EXCL_STOP */
1073 : }
1074 :
1075 265 : if (!*buffer) {
1076 : /* LCOV_EXCL_START */
1077 : log_fatal("Empty 'exclude' specification in '%s' at line %u\n", path, line);
1078 : exit(EXIT_FAILURE);
1079 : /* LCOV_EXCL_STOP */
1080 : }
1081 :
1082 265 : filter = filter_alloc_file(-1, buffer);
1083 265 : if (!filter) {
1084 : /* LCOV_EXCL_START */
1085 : log_fatal("Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1086 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1087 : exit(EXIT_FAILURE);
1088 : /* LCOV_EXCL_STOP */
1089 : }
1090 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1091 878 : } else if (strcmp(tag, "include") == 0) {
1092 : struct snapraid_filter* filter;
1093 :
1094 265 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1095 265 : if (ret < 0) {
1096 : /* LCOV_EXCL_START */
1097 : log_fatal("Invalid 'include' specification in '%s' at line %u\n", path, line);
1098 : exit(EXIT_FAILURE);
1099 : /* LCOV_EXCL_STOP */
1100 : }
1101 :
1102 265 : if (!*buffer) {
1103 : /* LCOV_EXCL_START */
1104 : log_fatal("Empty 'include' specification in '%s' at line %u\n", path, line);
1105 : exit(EXIT_FAILURE);
1106 : /* LCOV_EXCL_STOP */
1107 : }
1108 :
1109 265 : filter = filter_alloc_file(1, buffer);
1110 265 : if (!filter) {
1111 : /* LCOV_EXCL_START */
1112 : log_fatal("Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1113 : log_fatal("Filters using relative paths are not supported. Ensure to add an initial slash\n");
1114 : exit(EXIT_FAILURE);
1115 : /* LCOV_EXCL_STOP */
1116 : }
1117 265 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1118 613 : } else if (strcmp(tag, "autosave") == 0) {
1119 : char* e;
1120 :
1121 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1122 41 : if (ret < 0) {
1123 : /* LCOV_EXCL_START */
1124 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1125 : exit(EXIT_FAILURE);
1126 : /* LCOV_EXCL_STOP */
1127 : }
1128 :
1129 41 : if (!*buffer) {
1130 : /* LCOV_EXCL_START */
1131 : log_fatal("Empty 'autosave' specification in '%s' at line %u\n", path, line);
1132 : exit(EXIT_FAILURE);
1133 : /* LCOV_EXCL_STOP */
1134 : }
1135 :
1136 41 : state->autosave = strtoul(buffer, &e, 0);
1137 :
1138 41 : if (!e || *e) {
1139 : /* LCOV_EXCL_START */
1140 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1141 : exit(EXIT_FAILURE);
1142 : /* LCOV_EXCL_STOP */
1143 : }
1144 :
1145 : /* convert to GB */
1146 41 : state->autosave *= GIGA;
1147 572 : } else if (tag[0] == 0) {
1148 : /* allow empty lines */
1149 41 : } else if (tag[0] == '#') {
1150 41 : ret = sgetline(f, buffer, sizeof(buffer));
1151 41 : if (ret < 0) {
1152 : /* LCOV_EXCL_START */
1153 : log_fatal("Invalid comment in '%s' at line %u\n", path, line);
1154 : exit(EXIT_FAILURE);
1155 : /* LCOV_EXCL_STOP */
1156 : }
1157 : } else {
1158 : /* LCOV_EXCL_START */
1159 : log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1160 : exit(EXIT_FAILURE);
1161 : /* LCOV_EXCL_STOP */
1162 : }
1163 :
1164 : /* skip final spaces */
1165 6244 : sgetspace(f);
1166 :
1167 : /* next line */
1168 6244 : c = sgeteol(f);
1169 6244 : if (c == EOF) {
1170 265 : break;
1171 : }
1172 5979 : if (c != '\n') {
1173 : /* LCOV_EXCL_START */
1174 : log_fatal("Extra data in '%s' at line %u\n", path, line);
1175 : exit(EXIT_FAILURE);
1176 : /* LCOV_EXCL_STOP */
1177 : }
1178 5979 : ++line;
1179 5979 : }
1180 :
1181 265 : if (serror(f)) {
1182 : /* LCOV_EXCL_START */
1183 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
1184 : exit(EXIT_FAILURE);
1185 : /* LCOV_EXCL_STOP */
1186 : }
1187 :
1188 265 : sclose(f);
1189 :
1190 265 : state_config_check(state, path, filterlist_disk);
1191 :
1192 : /* select the default hash */
1193 265 : if (state->opt.force_murmur3) {
1194 1 : state->besthash = HASH_MURMUR3;
1195 264 : } else if (state->opt.force_spooky2) {
1196 1 : state->besthash = HASH_SPOOKY2;
1197 : } else {
1198 : #ifdef CONFIG_X86
1199 : if (sizeof(void*) == 4 && !raid_cpu_has_slowmult())
1200 : state->besthash = HASH_MURMUR3;
1201 : else
1202 263 : state->besthash = HASH_SPOOKY2;
1203 : #else
1204 : if (sizeof(void*) == 4)
1205 : state->besthash = HASH_MURMUR3;
1206 : else
1207 : state->besthash = HASH_SPOOKY2;
1208 : #endif
1209 : }
1210 :
1211 : /* by default use the best hash */
1212 265 : state->hash = state->besthash;
1213 :
1214 : /* by default use a random hash seed */
1215 265 : if (randomize(state->hashseed, HASH_MAX) != 0) {
1216 : /* LCOV_EXCL_START */
1217 : log_fatal("Failed to get random values.\n");
1218 : exit(EXIT_FAILURE);
1219 : /* LCOV_EXCL_STOP */
1220 : }
1221 :
1222 : /* no previous hash by default */
1223 265 : state->prevhash = HASH_UNDEFINED;
1224 :
1225 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1226 :
1227 265 : log_tag("blocksize:%u\n", state->block_size);
1228 1850 : for (i = state->disklist; i != 0; i = i->next) {
1229 1585 : struct snapraid_disk* disk = i->data;
1230 1585 : log_tag("data:%s:%s\n", disk->name, disk->dir);
1231 : }
1232 :
1233 265 : log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1234 1531 : for (l = 0; l < state->level; ++l)
1235 6330 : for (s = 0; s < state->parity[l].split_mac; ++s)
1236 5064 : log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
1237 265 : if (state->pool[0] != 0)
1238 41 : log_tag("pool:%s\n", state->pool);
1239 265 : if (state->share[0] != 0)
1240 41 : log_tag("share:%s\n", state->share);
1241 265 : if (state->autosave != 0)
1242 41 : log_tag("autosave:%" PRIu64 "\n", state->autosave);
1243 795 : for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1244 : char out[PATH_MAX];
1245 530 : struct snapraid_filter* filter = i->data;
1246 530 : log_tag("filter:%s\n", filter_type(filter, out, sizeof(out)));
1247 : }
1248 265 : if (state->filter_hidden)
1249 0 : log_tag("filter:nohidden:\n");
1250 265 : log_flush();
1251 : }
1252 :
1253 : /**
1254 : * Find a disk by name.
1255 : */
1256 7047 : static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1257 : {
1258 : tommy_node* i;
1259 :
1260 24588 : for (i = state->disklist; i != 0; i = i->next) {
1261 24580 : struct snapraid_disk* disk = i->data;
1262 24580 : if (strcmp(disk->name, name) == 0)
1263 7039 : return disk;
1264 : }
1265 :
1266 8 : if (state->no_conf) {
1267 : /* without a configuration file, add disks automatically */
1268 : struct snapraid_disk* disk;
1269 :
1270 6 : disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1271 :
1272 6 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1273 :
1274 6 : return disk;
1275 : }
1276 :
1277 2 : return 0;
1278 : }
1279 :
1280 : /**
1281 : * Find a disk by UUID.
1282 : */
1283 2 : static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1284 : {
1285 : tommy_node* i;
1286 2 : struct snapraid_disk* found = 0;
1287 :
1288 : /* special test case to find the first matching UUID */
1289 : /* when testing UUID are all equal or not supported */
1290 : /* and we should handle this case specifically */
1291 2 : if (state->opt.match_first_uuid)
1292 2 : return state->disklist->data;
1293 :
1294 : /* LCOV_EXCL_START */
1295 : /* never find an empty uuid */
1296 : if (!*uuid)
1297 : return 0;
1298 :
1299 : for (i = state->disklist; i != 0; i = i->next) {
1300 : struct snapraid_disk* disk = i->data;
1301 : if (strcmp(disk->uuid, uuid) == 0) {
1302 : /* never match duplicate UUID */
1303 : if (found)
1304 : return 0;
1305 : found = disk;
1306 : }
1307 : }
1308 :
1309 : return found;
1310 : /* LCOV_EXCL_STOP */
1311 : }
1312 :
1313 : /**
1314 : * Update the disk mapping if required.
1315 : */
1316 260 : static void state_map(struct snapraid_state* state)
1317 : {
1318 : unsigned hole;
1319 : tommy_node* i;
1320 : unsigned uuid_mismatch;
1321 : unsigned diskcount;
1322 : unsigned l, s;
1323 :
1324 : /* remove all the mapping without a disk */
1325 : /* this happens when a disk is removed from the configuration file */
1326 : /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1327 : /* when saving the content file, but we keep this code to import older content files. */
1328 1917 : for (i = state->maplist; i != 0; ) {
1329 1397 : struct snapraid_map* map = i->data;
1330 : struct snapraid_disk* disk;
1331 :
1332 1397 : disk = find_disk_by_name(state, map->name);
1333 :
1334 : /* go to the next mapping before removing */
1335 1397 : i = i->next;
1336 :
1337 1397 : if (disk == 0) {
1338 : /* disk not found, remove the mapping */
1339 0 : tommy_list_remove_existing(&state->maplist, &map->node);
1340 0 : map_free(map);
1341 : }
1342 : }
1343 :
1344 : /* maps each unmapped disk present in the configuration file in the first available hole */
1345 : /* this happens when you add disks for the first time in the configuration file */
1346 260 : hole = 0; /* first position to try */
1347 1815 : for (i = state->disklist; i != 0; i = i->next) {
1348 1555 : struct snapraid_disk* disk = i->data;
1349 : struct snapraid_map* map;
1350 : tommy_node* j;
1351 :
1352 : /* check if the disk is already mapped */
1353 5430 : for (j = state->maplist; j != 0; j = j->next) {
1354 5272 : map = j->data;
1355 5272 : if (strcmp(disk->name, map->name) == 0) {
1356 : /* mapping found */
1357 1397 : break;
1358 : }
1359 : }
1360 1555 : if (j != 0) {
1361 : /* mapping is present, then copy the free blocks into to disk */
1362 1397 : disk->total_blocks = map->total_blocks;
1363 1397 : disk->free_blocks = map->free_blocks;
1364 1397 : continue;
1365 : }
1366 :
1367 : /* mapping not found, search for an hole */
1368 : while (1) {
1369 1100 : for (j = state->maplist; j != 0; j = j->next) {
1370 942 : map = j->data;
1371 942 : if (map->position == hole) {
1372 : /* position already used */
1373 158 : break;
1374 : }
1375 : }
1376 316 : if (j == 0) {
1377 : /* hole found */
1378 158 : break;
1379 : }
1380 :
1381 : /* try with the next one */
1382 158 : ++hole;
1383 158 : }
1384 :
1385 : /* insert the new mapping */
1386 158 : map = map_alloc(disk->name, hole, 0, 0, "");
1387 :
1388 158 : tommy_list_insert_tail(&state->maplist, &map->node, map);
1389 : }
1390 :
1391 : /* without configuration don't check for number of data disks or uuid changes */
1392 260 : if (state->no_conf)
1393 1 : return;
1394 :
1395 : /* counter for the number of UUID mismatches */
1396 259 : uuid_mismatch = 0;
1397 :
1398 : /* check if mapping match the disk uuid */
1399 259 : if (!state->opt.skip_disk_access) {
1400 1626 : for (i = state->maplist; i != 0; i = i->next) {
1401 1393 : struct snapraid_map* map = i->data;
1402 : struct snapraid_disk* disk;
1403 :
1404 1393 : disk = find_disk_by_name(state, map->name);
1405 1393 : if (disk == 0) {
1406 : /* LCOV_EXCL_START */
1407 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1408 : os_abort();
1409 : /* LCOV_EXCL_STOP */
1410 : }
1411 :
1412 1393 : if (disk->has_unsupported_uuid) {
1413 : /* if uuid is not available, skip this one */
1414 1 : continue;
1415 : }
1416 :
1417 : /* if the uuid is changed */
1418 1392 : if (strcmp(disk->uuid, map->uuid) != 0) {
1419 : /* mark the disk as with an UUID change */
1420 113 : disk->has_different_uuid = 1;
1421 :
1422 : /* if the previous uuid is available */
1423 113 : if (map->uuid[0] != 0) {
1424 : /* count the number of uuid change */
1425 4 : ++uuid_mismatch;
1426 4 : log_fatal("UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1427 : } else {
1428 : /* no message here, because having a disk without */
1429 : /* UUID is the normal state of an empty disk */
1430 109 : disk->had_empty_uuid = 1;
1431 : }
1432 :
1433 : /* update the uuid in the mapping, */
1434 113 : pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1435 :
1436 : /* write the new state with the new uuid */
1437 113 : state->need_write = 1;
1438 : }
1439 : }
1440 : }
1441 :
1442 : /* check the parity uuid */
1443 259 : if (!state->opt.skip_parity_access) {
1444 1273 : for (l = 0; l < state->level; ++l) {
1445 5260 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1446 : char uuid[UUID_MAX];
1447 : int ret;
1448 :
1449 4208 : ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
1450 4208 : if (ret != 0) {
1451 : /* uuid not available, just ignore */
1452 4 : continue;
1453 : }
1454 :
1455 : /* if the uuid is changed */
1456 4204 : if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1457 : /* if the previous uuid is available */
1458 184 : if (state->parity[l].split_map[s].uuid[0] != 0) {
1459 : /* count the number of uuid change */
1460 0 : ++uuid_mismatch;
1461 0 : log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1462 : }
1463 :
1464 : /* update the uuid */
1465 184 : pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1466 :
1467 : /* write the new state with the new uuid */
1468 184 : state->need_write = 1;
1469 : }
1470 : }
1471 : }
1472 : }
1473 :
1474 259 : if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1475 : /* LCOV_EXCL_START */
1476 : log_fatal("Too many disks have UUID changed from the latest 'sync'.\n");
1477 : log_fatal("If this happens because you really replaced them,\n");
1478 : log_fatal("you can '%s' anyway, using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1479 : log_fatal("Instead, it's possible that you messed up the disk mount points,\n");
1480 : log_fatal("and you have to restore the mount points at the state of the latest sync.\n");
1481 : exit(EXIT_FAILURE);
1482 : /* LCOV_EXCL_STOP */
1483 : }
1484 :
1485 : /* count the number of data disks, including holes left after removing some */
1486 259 : diskcount = 0;
1487 1808 : for (i = state->maplist; i != 0; i = i->next) {
1488 1549 : struct snapraid_map* map = i->data;
1489 :
1490 1549 : if (map->position + 1 > diskcount)
1491 1367 : diskcount = map->position + 1;
1492 : }
1493 :
1494 : /* ensure to don't go over the limit of the RAID engine */
1495 259 : if (diskcount > RAID_DATA_MAX) {
1496 : /* LCOV_EXCL_START */
1497 : log_fatal("Too many data disks. No more than %u.\n", RAID_DATA_MAX);
1498 : exit(EXIT_FAILURE);
1499 : /* LCOV_EXCL_STOP */
1500 : }
1501 :
1502 : /* now count the real number of data disks, excluding holes left after removing some */
1503 259 : diskcount = tommy_list_count(&state->maplist);
1504 :
1505 : /* recommend number of parities */
1506 259 : if (!state->opt.no_warnings) {
1507 : /* intentionally use log_fatal() instead of log_error() to give more visibility at the warning */
1508 1 : if (diskcount >= 36 && state->level < 6) {
1509 0 : log_fatal("WARNING! With %u disks it's recommended to use six parity levels.\n", diskcount);
1510 1 : } else if (diskcount >= 29 && state->level < 5) {
1511 0 : log_fatal("WARNING! With %u disks it's recommended to use five parity levels.\n", diskcount);
1512 1 : } else if (diskcount >= 22 && state->level < 4) {
1513 0 : log_fatal("WARNING! With %u disks it's recommended to use four parity levels.\n", diskcount);
1514 1 : } else if (diskcount >= 15 && state->level < 3) {
1515 0 : log_fatal("WARNING! With %u disks it's recommended to use three parity levels.\n", diskcount);
1516 1 : } else if (diskcount >= 5 && state->level < 2) {
1517 0 : log_fatal("WARNING! With %u disks it's recommended to use two parity levels.\n", diskcount);
1518 : }
1519 : }
1520 : }
1521 :
1522 173 : void state_refresh(struct snapraid_state* state)
1523 : {
1524 : tommy_node* i;
1525 : unsigned l, s;
1526 :
1527 : /* for all disks */
1528 1209 : for (i = state->maplist; i != 0; i = i->next) {
1529 1036 : struct snapraid_map* map = i->data;
1530 : struct snapraid_disk* disk;
1531 : uint64_t total_space;
1532 : uint64_t free_space;
1533 : int ret;
1534 :
1535 1036 : disk = find_disk_by_name(state, map->name);
1536 1036 : if (disk == 0) {
1537 : /* LCOV_EXCL_START */
1538 : log_fatal("Internal inconsistency for mapping '%s'\n", map->name);
1539 : os_abort();
1540 : /* LCOV_EXCL_STOP */
1541 : }
1542 :
1543 1036 : ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space);
1544 1036 : if (ret != 0) {
1545 : /* LCOV_EXCL_START */
1546 : log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1547 : exit(EXIT_FAILURE);
1548 : /* LCOV_EXCL_STOP */
1549 : }
1550 :
1551 : /* set the new free blocks */
1552 1036 : map->total_blocks = total_space / state->block_size;
1553 1036 : map->free_blocks = free_space / state->block_size;
1554 :
1555 : /* also update the disk info */
1556 1036 : disk->total_blocks = map->total_blocks;
1557 1036 : disk->free_blocks = map->free_blocks;
1558 : }
1559 :
1560 : /* for all parities */
1561 1141 : for (l = 0; l < state->level; ++l) {
1562 : /* set the new free blocks */
1563 968 : state->parity[l].total_blocks = 0;
1564 968 : state->parity[l].free_blocks = 0;
1565 :
1566 4840 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1567 : uint64_t total_space;
1568 : uint64_t free_space;
1569 : int ret;
1570 :
1571 3872 : ret = fsinfo(state->parity[l].split_map[s].path, 0, 0, &total_space, &free_space);
1572 3872 : if (ret != 0) {
1573 : /* LCOV_EXCL_START */
1574 : log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
1575 : exit(EXIT_FAILURE);
1576 : /* LCOV_EXCL_STOP */
1577 : }
1578 :
1579 : /* add the new free blocks */
1580 3872 : state->parity[l].total_blocks += total_space / state->block_size;
1581 3872 : state->parity[l].free_blocks += free_space / state->block_size;
1582 : }
1583 : }
1584 :
1585 : /* note what we don't set need_write = 1, because we don't want */
1586 : /* to update the content file only for the free space info. */
1587 173 : }
1588 :
1589 : /**
1590 : * Check the content.
1591 : */
1592 253 : static void state_content_check(struct snapraid_state* state, const char* path)
1593 : {
1594 : tommy_node* i;
1595 :
1596 : /* check that any map has different name and position */
1597 1766 : for (i = state->maplist; i != 0; i = i->next) {
1598 1513 : struct snapraid_map* map = i->data;
1599 : tommy_node* j;
1600 5283 : for (j = i->next; j != 0; j = j->next) {
1601 3770 : struct snapraid_map* other = j->data;
1602 3770 : if (strcmp(map->name, other->name) == 0) {
1603 : /* LCOV_EXCL_START */
1604 : log_fatal("Colliding 'map' disk specification in '%s'\n", path);
1605 : exit(EXIT_FAILURE);
1606 : /* LCOV_EXCL_STOP */
1607 : }
1608 3770 : if (map->position == other->position) {
1609 : /* LCOV_EXCL_START */
1610 : log_fatal("Colliding 'map' index specification in '%s'\n", path);
1611 : exit(EXIT_FAILURE);
1612 : /* LCOV_EXCL_STOP */
1613 : }
1614 : }
1615 : }
1616 253 : }
1617 :
1618 : /**
1619 : * Check if the position is REQUIRED, or we can completely clear it from the state.
1620 : *
1621 : * Note that position with only DELETED blocks are discarged.
1622 : */
1623 756099 : static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1624 : {
1625 : tommy_node* i;
1626 :
1627 : /* check for each disk */
1628 802725 : for (i = state->disklist; i != 0; i = i->next) {
1629 802725 : struct snapraid_disk* disk = i->data;
1630 802725 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1631 :
1632 : /* if we have at least one file, the position is needed */
1633 802725 : if (block_has_file(block))
1634 756099 : return 1;
1635 : }
1636 :
1637 0 : return 0;
1638 : }
1639 :
1640 : /**
1641 : * Check if the info block is REQUIREQ.
1642 : *
1643 : * This is used to ensure that we keep the last check used for scrubbing.
1644 : * and that we add it when importing old context files.
1645 : *
1646 : * Note that you can have position without info blocks, for example
1647 : * if all the blocks are not synced.
1648 : *
1649 : * Note also that not requiring an info block, doesn't mean that if present it
1650 : * can be discarded.
1651 : */
1652 1547923 : static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1653 : {
1654 : tommy_node* i;
1655 :
1656 : /* check for each disk */
1657 2048488 : for (i = state->disklist; i != 0; i = i->next) {
1658 1989723 : struct snapraid_disk* disk = i->data;
1659 1989723 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1660 :
1661 : /* if we have at least one synced file, the info is required */
1662 1989723 : if (block_state_get(block) == BLOCK_STATE_BLK)
1663 1489158 : return 1;
1664 : }
1665 :
1666 58765 : return 0;
1667 : }
1668 :
1669 0 : static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1670 : {
1671 : tommy_node* i;
1672 :
1673 : /* check for each disk if block is really used */
1674 0 : for (i = state->disklist; i != 0; i = i->next) {
1675 0 : struct snapraid_disk* disk = i->data;
1676 0 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1677 :
1678 : /* if the block is deleted */
1679 0 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1680 : /* set it to empty */
1681 0 : fs_deallocate(disk, pos);
1682 : }
1683 : }
1684 0 : }
1685 :
1686 : /**
1687 : * Check if a block position in a disk is deleted.
1688 : */
1689 4529621 : static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
1690 : {
1691 4529621 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1692 :
1693 4529621 : return block_state_get(block) == BLOCK_STATE_DELETED;
1694 : }
1695 :
1696 : /**
1697 : * Flush the file checking the final CRC.
1698 : * We exploit the fact that the CRC is always stored in the last 4 bytes.
1699 : */
1700 1 : static void decoding_error(const char* path, STREAM* f)
1701 : {
1702 : unsigned char buf[4];
1703 : uint32_t crc_stored;
1704 : uint32_t crc_computed;
1705 :
1706 1 : if (seof(f)) {
1707 : /* LCOV_EXCL_START */
1708 : log_fatal("Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1709 : log_fatal("This content file is truncated. Use an alternate copy.\n");
1710 : exit(EXIT_FAILURE);
1711 : /* LCOV_EXCL_STOP */
1712 : }
1713 :
1714 1 : if (serror(f)) {
1715 : /* LCOV_EXCL_START */
1716 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1717 : exit(EXIT_FAILURE);
1718 : /* LCOV_EXCL_STOP */
1719 : }
1720 :
1721 1 : log_fatal("Decoding error in '%s' at offset %" PRIi64 "\n", path, stell(f));
1722 :
1723 1 : if (sdeplete(f, buf) != 0) {
1724 : /* LCOV_EXCL_START */
1725 : log_fatal("Error flushing the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1726 : exit(EXIT_FAILURE);
1727 : /* LCOV_EXCL_STOP */
1728 : }
1729 :
1730 : /* get the stored crc from the last four bytes */
1731 1 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
1732 :
1733 : /* get the computed crc */
1734 1 : crc_computed = scrc(f);
1735 :
1736 : /* adjust the stored crc to include itself */
1737 1 : crc_stored = crc32c(crc_stored, buf, 4);
1738 :
1739 1 : if (crc_computed != crc_stored) {
1740 1 : log_fatal("Mismatching CRC in '%s'\n", path);
1741 1 : log_fatal("This content file is damaged! Use an alternate copy.\n");
1742 1 : exit(EXIT_FAILURE);
1743 : } else {
1744 0 : log_fatal("The file CRC is correct!\n");
1745 : }
1746 0 : }
1747 :
1748 254 : static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
1749 : {
1750 : block_off_t blockmax;
1751 : unsigned count_file;
1752 : unsigned count_hardlink;
1753 : unsigned count_symlink;
1754 : unsigned count_dir;
1755 : int crc_checked;
1756 : char buffer[PATH_MAX];
1757 : int ret;
1758 : tommy_array disk_mapping;
1759 : uint32_t mapping_max;
1760 :
1761 254 : blockmax = 0;
1762 254 : count_file = 0;
1763 254 : count_hardlink = 0;
1764 254 : count_symlink = 0;
1765 254 : count_dir = 0;
1766 254 : crc_checked = 0;
1767 254 : mapping_max = 0;
1768 254 : tommy_array_init(&disk_mapping);
1769 :
1770 254 : ret = sread(f, buffer, 12);
1771 254 : if (ret < 0) {
1772 : /* LCOV_EXCL_START */
1773 : decoding_error(path, f);
1774 : log_fatal("Invalid header!\n");
1775 : os_abort();
1776 : /* LCOV_EXCL_STOP */
1777 : }
1778 :
1779 : /*
1780 : * File format versions:
1781 : * - SNAPCNT1/SnapRAID 4.0 First version.
1782 : * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
1783 : * The previous 'm' entry is now deprecated, but supported for importing.
1784 : * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
1785 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
1786 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
1787 : * The previous 'P' entry is now deprecated, but supported for importing.
1788 : */
1789 254 : if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
1790 254 : && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
1791 254 : && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
1792 : ) {
1793 : /* LCOV_EXCL_START */
1794 : if (memcmp(buffer, "SNAPCNT", 7) != 0) {
1795 : decoding_error(path, f);
1796 : log_fatal("Invalid header!\n");
1797 : os_abort();
1798 : } else {
1799 : log_fatal("The content file '%s' was generated with a newer version of SnapRAID!\n", path);
1800 : exit(EXIT_FAILURE);
1801 : }
1802 : /* LCOV_EXCL_STOP */
1803 : }
1804 :
1805 : while (1) {
1806 : int c;
1807 :
1808 : /* read the command */
1809 3735611 : c = sgetc(f);
1810 3735611 : if (c == EOF) {
1811 253 : break;
1812 : }
1813 :
1814 3735358 : if (c == 'f') {
1815 : /* file */
1816 : char sub[PATH_MAX];
1817 : uint64_t v_size;
1818 : uint64_t v_mtime_sec;
1819 : uint32_t v_mtime_nsec;
1820 : uint64_t v_inode;
1821 : uint32_t v_idx;
1822 : struct snapraid_file* file;
1823 : struct snapraid_disk* disk;
1824 : uint32_t mapping;
1825 :
1826 3611762 : ret = sgetb32(f, &mapping);
1827 3611762 : if (ret < 0 || mapping >= mapping_max) {
1828 : /* LCOV_EXCL_START */
1829 : decoding_error(path, f);
1830 : log_fatal("Internal inconsistency in mapping index!\n");
1831 : os_abort();
1832 : /* LCOV_EXCL_STOP */
1833 : }
1834 3611762 : disk = tommy_array_get(&disk_mapping, mapping);
1835 :
1836 3611762 : ret = sgetb64(f, &v_size);
1837 3611762 : if (ret < 0) {
1838 : /* LCOV_EXCL_START */
1839 : decoding_error(path, f);
1840 : os_abort();
1841 : /* LCOV_EXCL_STOP */
1842 : }
1843 :
1844 3611762 : if (state->block_size == 0) {
1845 : /* LCOV_EXCL_START */
1846 : decoding_error(path, f);
1847 : log_fatal("Internal incosistency due zero blocksize!\n");
1848 : exit(EXIT_FAILURE);
1849 : /* LCOV_EXCL_STOP */
1850 : }
1851 :
1852 : /* check for impossible file size to avoid to crash for a too big allocation */
1853 3611762 : if (v_size / state->block_size > blockmax) {
1854 : /* LCOV_EXCL_START */
1855 : decoding_error(path, f);
1856 : log_fatal("Internal inconsistency in file size too big!\n");
1857 : os_abort();
1858 : /* LCOV_EXCL_STOP */
1859 : }
1860 :
1861 3611762 : ret = sgetb64(f, &v_mtime_sec);
1862 3611762 : if (ret < 0) {
1863 : /* LCOV_EXCL_START */
1864 : decoding_error(path, f);
1865 : os_abort();
1866 : /* LCOV_EXCL_STOP */
1867 : }
1868 :
1869 3611762 : ret = sgetb32(f, &v_mtime_nsec);
1870 3611762 : if (ret < 0) {
1871 : /* LCOV_EXCL_START */
1872 : decoding_error(path, f);
1873 : os_abort();
1874 : /* LCOV_EXCL_STOP */
1875 : }
1876 :
1877 : /* STAT_NSEC_INVALID is encoded as 0 */
1878 3611762 : if (v_mtime_nsec == 0)
1879 0 : v_mtime_nsec = STAT_NSEC_INVALID;
1880 : else
1881 3611762 : --v_mtime_nsec;
1882 :
1883 3611762 : ret = sgetb64(f, &v_inode);
1884 3611762 : if (ret < 0) {
1885 : /* LCOV_EXCL_START */
1886 : decoding_error(path, f);
1887 : os_abort();
1888 : /* LCOV_EXCL_STOP */
1889 : }
1890 :
1891 3611762 : ret = sgetbs(f, sub, sizeof(sub));
1892 3611762 : if (ret < 0) {
1893 : /* LCOV_EXCL_START */
1894 : decoding_error(path, f);
1895 : os_abort();
1896 : /* LCOV_EXCL_STOP */
1897 : }
1898 3611762 : if (!*sub) {
1899 : /* LCOV_EXCL_START */
1900 : decoding_error(path, f);
1901 : log_fatal("Internal inconsistency for null file!\n");
1902 : os_abort();
1903 : /* LCOV_EXCL_STOP */
1904 : }
1905 :
1906 : /* allocate the file */
1907 3611762 : file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
1908 :
1909 : /* insert the file in the file containers */
1910 3611762 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
1911 3611762 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
1912 3611762 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
1913 3611762 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
1914 :
1915 : /* read all the blocks */
1916 3611762 : v_idx = 0;
1917 10855957 : while (v_idx < file->blockmax) {
1918 : block_off_t v_pos;
1919 : uint32_t v_count;
1920 :
1921 : /* get the "subcommand */
1922 3632433 : c = sgetc(f);
1923 :
1924 3632433 : ret = sgetb32(f, &v_pos);
1925 3632433 : if (ret < 0) {
1926 : /* LCOV_EXCL_START */
1927 : decoding_error(path, f);
1928 : os_abort();
1929 : /* LCOV_EXCL_STOP */
1930 : }
1931 :
1932 3632433 : ret = sgetb32(f, &v_count);
1933 3632433 : if (ret < 0) {
1934 : /* LCOV_EXCL_START */
1935 : decoding_error(path, f);
1936 : os_abort();
1937 : /* LCOV_EXCL_STOP */
1938 : }
1939 :
1940 3632433 : if (v_idx + v_count > file->blockmax) {
1941 : /* LCOV_EXCL_START */
1942 : decoding_error(path, f);
1943 : log_fatal("Internal inconsistency in block number!\n");
1944 : os_abort();
1945 : /* LCOV_EXCL_STOP */
1946 : }
1947 :
1948 3632433 : if (v_pos + v_count > blockmax) {
1949 : /* LCOV_EXCL_START */
1950 : decoding_error(path, f);
1951 : log_fatal("Internal inconsistency in block size %u/%u!\n", blockmax, v_pos + v_count);
1952 : os_abort();
1953 : /* LCOV_EXCL_START */
1954 : }
1955 :
1956 : /* fill the blocks in the run */
1957 : while (v_count) {
1958 : struct snapraid_block* block = fs_file2block_get(file, v_idx);
1959 :
1960 : switch (c) {
1961 : case 'b' :
1962 : block_state_set(block, BLOCK_STATE_BLK);
1963 : break;
1964 : case 'n' :
1965 : /* deprecated NEW blocks are converted to CHG ones */
1966 : block_state_set(block, BLOCK_STATE_CHG);
1967 : break;
1968 : case 'g' :
1969 : block_state_set(block, BLOCK_STATE_CHG);
1970 : break;
1971 : case 'p' :
1972 : block_state_set(block, BLOCK_STATE_REP);
1973 : break;
1974 : default :
1975 : /* LCOV_EXCL_START */
1976 : decoding_error(path, f);
1977 : log_fatal("Invalid block type!\n");
1978 : os_abort();
1979 : /* LCOV_EXCL_STOP */
1980 : }
1981 :
1982 : /* read the hash only for 'blk/chg/rep', and not for 'new' */
1983 8896100 : if (c != 'n') {
1984 8896100 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
1985 8896100 : if (ret < 0) {
1986 : /* LCOV_EXCL_START */
1987 : decoding_error(path, f);
1988 : os_abort();
1989 : /* LCOV_EXCL_STOP */
1990 : }
1991 : } else {
1992 : /* set the ZERO hash for deprecated NEW blocks */
1993 0 : hash_zero_set(block->hash);
1994 : }
1995 :
1996 : /* if the block contains a hash of past data */
1997 : /* and we are clearing such indeterminate hashes */
1998 8896100 : if (state->clear_past_hash
1999 2542004 : && block_has_past_hash(block)
2000 : ) {
2001 : /* set the hash value to INVALID */
2002 28006 : hash_invalid_set(block->hash);
2003 : }
2004 :
2005 : /* if we are disabling the copy optimization */
2006 : /* we want also to clear any already previously stored information */
2007 : /* in other sync commands */
2008 : /* note that this is required only in sync, and we detect */
2009 : /* this using the clear_past_hash flag */
2010 8896100 : if (state->clear_past_hash
2011 2542004 : && state->opt.force_nocopy
2012 14195 : && block_state_get(block) == BLOCK_STATE_REP
2013 : ) {
2014 : /* set the hash value to INVALID */
2015 1 : hash_invalid_set(block->hash);
2016 : /* convert from REP to CHG block */
2017 1 : block_state_set(block, BLOCK_STATE_CHG);
2018 : }
2019 :
2020 : /* if we want a full reallocation, marks block as invalid parity */
2021 : /* note that we do this after the force_nocopy option */
2022 : /* to avoid to mixup the two things */
2023 8896100 : if (state->opt.force_realloc
2024 27826 : && block_state_get(block) == BLOCK_STATE_BLK) {
2025 : /* convert from BLK to REP */
2026 27826 : block_state_set(block, BLOCK_STATE_REP);
2027 : }
2028 :
2029 : /* set the parity association */
2030 8896100 : fs_allocate(disk, v_pos, file, v_idx);
2031 :
2032 : /* go to the next block */
2033 8896100 : ++v_idx;
2034 8896100 : ++v_pos;
2035 8896100 : --v_count;
2036 : }
2037 : }
2038 :
2039 : /* stat */
2040 3611762 : ++count_file;
2041 123596 : } else if (c == 'i') {
2042 : /* "inf" command */
2043 : snapraid_info info;
2044 : uint32_t v_pos;
2045 : uint32_t v_oldest;
2046 :
2047 253 : ret = sgetb32(f, &v_oldest);
2048 253 : if (ret < 0) {
2049 : /* LCOV_EXCL_START */
2050 : decoding_error(path, f);
2051 : os_abort();
2052 : /* LCOV_EXCL_STOP */
2053 : }
2054 :
2055 253 : v_pos = 0;
2056 55763 : while (v_pos < blockmax) {
2057 : int bad;
2058 : int rehash;
2059 : int justsynced;
2060 : uint32_t t;
2061 : uint32_t flag;
2062 : uint32_t v_count;
2063 :
2064 55257 : ret = sgetb32(f, &v_count);
2065 55257 : if (ret < 0) {
2066 : /* LCOV_EXCL_START */
2067 : decoding_error(path, f);
2068 : os_abort();
2069 : /* LCOV_EXCL_STOP */
2070 : }
2071 :
2072 55257 : if (v_pos + v_count > blockmax) {
2073 : /* LCOV_EXCL_START */
2074 : decoding_error(path, f);
2075 : log_fatal("Internal inconsistency in info size %u/%u!\n", blockmax, v_pos + v_count);
2076 : os_abort();
2077 : /* LCOV_EXCL_STOP */
2078 : }
2079 :
2080 55257 : ret = sgetb32(f, &flag);
2081 55257 : if (ret < 0) {
2082 : /* LCOV_EXCL_START */
2083 : decoding_error(path, f);
2084 : os_abort();
2085 : /* LCOV_EXCL_STOP */
2086 : }
2087 :
2088 : /* if there is an info */
2089 55257 : if ((flag & 1) != 0) {
2090 : /* read the time */
2091 55242 : ret = sgetb32(f, &t);
2092 55242 : if (ret < 0) {
2093 : /* LCOV_EXCL_START */
2094 : decoding_error(path, f);
2095 : os_abort();
2096 : /* LCOV_EXCL_STOP */
2097 : }
2098 :
2099 : /* analyze the flags */
2100 55242 : bad = (flag & 2) != 0;
2101 55242 : rehash = (flag & 4) != 0;
2102 55242 : justsynced = (flag & 8) != 0;
2103 :
2104 55242 : if (rehash && state->prevhash == HASH_UNDEFINED) {
2105 : /* LCOV_EXCL_START */
2106 : decoding_error(path, f);
2107 : log_fatal("Internal inconsistency for missing previous checksum!\n");
2108 : os_abort();
2109 : /* LCOV_EXCL_STOP */
2110 : }
2111 :
2112 55242 : info = info_make(t + v_oldest, bad, rehash, justsynced);
2113 : } else {
2114 15 : info = 0;
2115 : }
2116 :
2117 1658437 : while (v_count) {
2118 : /* insert the info in the array */
2119 1547923 : info_set(&state->infoarr, v_pos, info);
2120 :
2121 : /* ensure that an info is present only for used positions */
2122 1547923 : if (fs_info_is_required(state, v_pos)) {
2123 1489158 : if (!info) {
2124 : /* LCOV_EXCL_START */
2125 : decoding_error(path, f);
2126 : log_fatal("Internal inconsistency for missing info!\n");
2127 : os_abort();
2128 : /* LCOV_EXCL_STOP */
2129 : }
2130 : } else {
2131 : /* extra info are accepted for backward compatibility */
2132 : /* they are discarded at the first write */
2133 : }
2134 :
2135 : /* go to next block */
2136 1547923 : ++v_pos;
2137 1547923 : --v_count;
2138 : }
2139 : }
2140 123343 : } else if (c == 'h') {
2141 : /* hole */
2142 : uint32_t v_pos;
2143 : struct snapraid_disk* disk;
2144 : uint32_t mapping;
2145 :
2146 1397 : ret = sgetb32(f, &mapping);
2147 1397 : if (ret < 0 || mapping >= mapping_max) {
2148 : /* LCOV_EXCL_START */
2149 : decoding_error(path, f);
2150 : log_fatal("Internal inconsistency in mapping index!\n");
2151 : os_abort();
2152 : /* LCOV_EXCL_STOP */
2153 : }
2154 1397 : disk = tommy_array_get(&disk_mapping, mapping);
2155 :
2156 1397 : v_pos = 0;
2157 4257 : while (v_pos < blockmax) {
2158 : uint32_t v_idx;
2159 : uint32_t v_count;
2160 : struct snapraid_file* deleted;
2161 :
2162 1463 : ret = sgetb32(f, &v_count);
2163 1463 : if (ret < 0) {
2164 : /* LCOV_EXCL_START */
2165 : decoding_error(path, f);
2166 : os_abort();
2167 : /* LCOV_EXCL_STOP */
2168 : }
2169 :
2170 1463 : if (v_pos + v_count > blockmax) {
2171 : /* LCOV_EXCL_START */
2172 : decoding_error(path, f);
2173 : log_fatal("Internal inconsistency in hole size %u/%u!\n", blockmax, v_pos + v_count);
2174 : os_abort();
2175 : /* LCOV_EXCL_STOP */
2176 : }
2177 :
2178 : /* get the sub-command */
2179 1463 : c = sgetc(f);
2180 :
2181 1463 : switch (c) {
2182 : case 'o' :
2183 : /* if it's a run of deleted blocks */
2184 :
2185 : /* allocate a fake deleted file */
2186 35 : deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2187 :
2188 : /* mark the file as deleted */
2189 35 : file_flag_set(deleted, FILE_IS_DELETED);
2190 :
2191 : /* insert it in the list of deleted files */
2192 35 : tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2193 :
2194 : /* process all blocks */
2195 35 : v_idx = 0;
2196 27956 : while (v_count) {
2197 27886 : struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2198 :
2199 : /* set the block as deleted */
2200 27886 : block_state_set(block, BLOCK_STATE_DELETED);
2201 :
2202 : /* read the hash */
2203 27886 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2204 27886 : if (ret < 0) {
2205 : /* LCOV_EXCL_START */
2206 : decoding_error(path, f);
2207 : os_abort();
2208 : /* LCOV_EXCL_STOP */
2209 : }
2210 :
2211 : /* if we are clearing indeterminate hashes */
2212 27886 : if (state->clear_past_hash) {
2213 : /* set the hash value to INVALID */
2214 9432 : hash_invalid_set(block->hash);
2215 : }
2216 :
2217 : /* insert the block in the block array */
2218 27886 : fs_allocate(disk, v_pos, deleted, v_idx);
2219 :
2220 : /* go to next block */
2221 27886 : ++v_pos;
2222 27886 : ++v_idx;
2223 27886 : --v_count;
2224 : }
2225 35 : break;
2226 : case 'O' :
2227 : /* go to the next run */
2228 1428 : v_pos += v_count;
2229 1428 : break;
2230 : default :
2231 : /* LCOV_EXCL_START */
2232 : decoding_error(path, f);
2233 : log_fatal("Invalid hole type!\n");
2234 : os_abort();
2235 : /* LCOV_EXCL_STOP */
2236 : }
2237 : }
2238 121946 : } else if (c == 's') {
2239 : /* symlink */
2240 : char sub[PATH_MAX];
2241 : char linkto[PATH_MAX];
2242 : struct snapraid_link* slink;
2243 : struct snapraid_disk* disk;
2244 : uint32_t mapping;
2245 :
2246 116468 : ret = sgetb32(f, &mapping);
2247 116468 : if (ret < 0 || mapping >= mapping_max) {
2248 : /* LCOV_EXCL_START */
2249 : decoding_error(path, f);
2250 : log_fatal("Internal inconsistency in mapping index!\n");
2251 : os_abort();
2252 : /* LCOV_EXCL_STOP */
2253 : }
2254 116468 : disk = tommy_array_get(&disk_mapping, mapping);
2255 :
2256 116468 : ret = sgetbs(f, sub, sizeof(sub));
2257 116468 : if (ret < 0) {
2258 : /* LCOV_EXCL_START */
2259 : decoding_error(path, f);
2260 : os_abort();
2261 : /* LCOV_EXCL_STOP */
2262 : }
2263 :
2264 116468 : if (!*sub) {
2265 : /* LCOV_EXCL_START */
2266 : decoding_error(path, f);
2267 : log_fatal("Internal inconsistency for null symlink!\n");
2268 : os_abort();
2269 : /* LCOV_EXCL_STOP */
2270 : }
2271 :
2272 116468 : ret = sgetbs(f, linkto, sizeof(linkto));
2273 116468 : if (ret < 0) {
2274 : /* LCOV_EXCL_START */
2275 : decoding_error(path, f);
2276 : os_abort();
2277 : /* LCOV_EXCL_STOP */
2278 : }
2279 :
2280 : /* allocate the link as symbolic link */
2281 116468 : slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2282 :
2283 : /* insert the link in the link containers */
2284 116468 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2285 116468 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2286 :
2287 : /* stat */
2288 116468 : ++count_symlink;
2289 5478 : } else if (c == 'a') {
2290 : /* hardlink */
2291 : char sub[PATH_MAX];
2292 : char linkto[PATH_MAX];
2293 : struct snapraid_link* slink;
2294 : struct snapraid_disk* disk;
2295 : uint32_t mapping;
2296 :
2297 597 : ret = sgetb32(f, &mapping);
2298 597 : if (ret < 0 || mapping >= mapping_max) {
2299 : /* LCOV_EXCL_START */
2300 : decoding_error(path, f);
2301 : log_fatal("Internal inconsistency in mapping index!\n");
2302 : os_abort();
2303 : /* LCOV_EXCL_STOP */
2304 : }
2305 597 : disk = tommy_array_get(&disk_mapping, mapping);
2306 :
2307 597 : ret = sgetbs(f, sub, sizeof(sub));
2308 597 : if (ret < 0) {
2309 : /* LCOV_EXCL_START */
2310 : decoding_error(path, f);
2311 : os_abort();
2312 : /* LCOV_EXCL_STOP */
2313 : }
2314 :
2315 597 : if (!*sub) {
2316 : /* LCOV_EXCL_START */
2317 : decoding_error(path, f);
2318 : log_fatal("Internal inconsistency for null hardlink!\n");
2319 : os_abort();
2320 : /* LCOV_EXCL_STOP */
2321 : }
2322 :
2323 597 : ret = sgetbs(f, linkto, sizeof(linkto));
2324 597 : if (ret < 0) {
2325 : /* LCOV_EXCL_START */
2326 : decoding_error(path, f);
2327 : os_abort();
2328 : /* LCOV_EXCL_STOP */
2329 : }
2330 :
2331 597 : if (!*linkto) {
2332 : /* LCOV_EXCL_START */
2333 : decoding_error(path, f);
2334 : log_fatal("Internal inconsistency for empty hardlink '%s'!\n", sub);
2335 : os_abort();
2336 : /* LCOV_EXCL_STOP */
2337 : }
2338 :
2339 : /* allocate the link as hard link */
2340 597 : slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2341 :
2342 : /* insert the link in the link containers */
2343 597 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2344 597 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2345 :
2346 : /* stat */
2347 597 : ++count_hardlink;
2348 4881 : } else if (c == 'r') {
2349 : /* dir */
2350 : char sub[PATH_MAX];
2351 : struct snapraid_dir* dir;
2352 : struct snapraid_disk* disk;
2353 : uint32_t mapping;
2354 :
2355 790 : ret = sgetb32(f, &mapping);
2356 790 : if (ret < 0 || mapping >= mapping_max) {
2357 : /* LCOV_EXCL_START */
2358 : decoding_error(path, f);
2359 : log_fatal("Internal inconsistency in mapping index!\n");
2360 : os_abort();
2361 : /* LCOV_EXCL_STOP */
2362 : }
2363 790 : disk = tommy_array_get(&disk_mapping, mapping);
2364 :
2365 790 : ret = sgetbs(f, sub, sizeof(sub));
2366 790 : if (ret < 0) {
2367 : /* LCOV_EXCL_START */
2368 : decoding_error(path, f);
2369 : os_abort();
2370 : /* LCOV_EXCL_STOP */
2371 : }
2372 :
2373 790 : if (!*sub) {
2374 : /* LCOV_EXCL_START */
2375 : decoding_error(path, f);
2376 : log_fatal("Internal inconsistency for null dir!\n");
2377 : os_abort();
2378 : /* LCOV_EXCL_STOP */
2379 : }
2380 :
2381 : /* allocate the dir */
2382 790 : dir = dir_alloc(sub);
2383 :
2384 : /* insert the dir in the dir containers */
2385 790 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2386 790 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2387 :
2388 : /* stat */
2389 790 : ++count_dir;
2390 4091 : } else if (c == 'c') {
2391 : /* get the subcommand */
2392 254 : c = sgetc(f);
2393 :
2394 254 : switch (c) {
2395 : case 'u' :
2396 139 : state->hash = HASH_MURMUR3;
2397 139 : break;
2398 : case 'k' :
2399 115 : state->hash = HASH_SPOOKY2;
2400 115 : break;
2401 : default :
2402 : /* LCOV_EXCL_START */
2403 : decoding_error(path, f);
2404 : log_fatal("Invalid checksum!\n");
2405 : os_abort();
2406 : /* LCOV_EXCL_STOP */
2407 : }
2408 :
2409 : /* read the seed */
2410 254 : ret = sread(f, state->hashseed, HASH_MAX);
2411 254 : if (ret < 0) {
2412 : /* LCOV_EXCL_START */
2413 : decoding_error(path, f);
2414 : os_abort();
2415 : /* LCOV_EXCL_STOP */
2416 : }
2417 3837 : } else if (c == 'C') {
2418 : /* get the sub-command */
2419 4 : c = sgetc(f);
2420 :
2421 4 : switch (c) {
2422 : case 'u' :
2423 4 : state->prevhash = HASH_MURMUR3;
2424 4 : break;
2425 : case 'k' :
2426 0 : state->prevhash = HASH_SPOOKY2;
2427 0 : break;
2428 : default :
2429 : /* LCOV_EXCL_START */
2430 : decoding_error(path, f);
2431 : log_fatal("Invalid checksum!\n");
2432 : os_abort();
2433 : /* LCOV_EXCL_STOP */
2434 : }
2435 :
2436 : /* read the seed */
2437 4 : ret = sread(f, state->prevhashseed, HASH_MAX);
2438 4 : if (ret < 0) {
2439 : /* LCOV_EXCL_START */
2440 : decoding_error(path, f);
2441 : os_abort();
2442 : /* LCOV_EXCL_STOP */
2443 : }
2444 3833 : } else if (c == 'z') {
2445 : uint32_t block_size;
2446 :
2447 254 : ret = sgetb32(f, &block_size);
2448 254 : if (ret < 0) {
2449 : /* LCOV_EXCL_START */
2450 : decoding_error(path, f);
2451 : os_abort();
2452 : /* LCOV_EXCL_STOP */
2453 : }
2454 :
2455 254 : if (block_size == 0) {
2456 : /* LCOV_EXCL_START */
2457 : decoding_error(path, f);
2458 : log_fatal("Zero 'blocksize' specification in the content file!\n");
2459 : exit(EXIT_FAILURE);
2460 : /* LCOV_EXCL_STOP */
2461 : }
2462 :
2463 : /* without configuration, auto assign the block size */
2464 254 : if (state->no_conf) {
2465 1 : state->block_size = block_size;
2466 : }
2467 :
2468 254 : if (block_size != state->block_size) {
2469 : /* LCOV_EXCL_START */
2470 : decoding_error(path, f);
2471 : log_fatal("Mismatching 'blocksize' specification in the content file!\n");
2472 : log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2473 : exit(EXIT_FAILURE);
2474 : /* LCOV_EXCL_STOP */
2475 : }
2476 3579 : } else if (c == 'y') {
2477 : uint32_t hash_size;
2478 :
2479 254 : ret = sgetb32(f, &hash_size);
2480 254 : if (ret < 0) {
2481 : /* LCOV_EXCL_START */
2482 : decoding_error(path, f);
2483 : os_abort();
2484 : /* LCOV_EXCL_STOP */
2485 : }
2486 :
2487 254 : if (hash_size < 2 || hash_size > HASH_MAX) {
2488 : /* LCOV_EXCL_START */
2489 : decoding_error(path, f);
2490 : log_fatal("Invalid 'hashsize' specification in the content file!\n");
2491 : exit(EXIT_FAILURE);
2492 : /* LCOV_EXCL_STOP */
2493 : }
2494 :
2495 : /* without configuration, auto assign the block size */
2496 254 : if (state->no_conf) {
2497 1 : BLOCK_HASH_SIZE = hash_size;
2498 : }
2499 :
2500 254 : if ((int)hash_size != BLOCK_HASH_SIZE) {
2501 : /* LCOV_EXCL_START */
2502 : decoding_error(path, f);
2503 : log_fatal("Mismatching 'hashsize' specification in the content file!\n");
2504 : log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2505 : exit(EXIT_FAILURE);
2506 : /* LCOV_EXCL_STOP */
2507 : }
2508 3325 : } else if (c == 'x') {
2509 254 : ret = sgetb32(f, &blockmax);
2510 254 : if (ret < 0) {
2511 : /* LCOV_EXCL_START */
2512 : decoding_error(path, f);
2513 : os_abort();
2514 : /* LCOV_EXCL_STOP */
2515 : }
2516 4474 : } else if (c == 'm' || c == 'M') {
2517 : struct snapraid_map* map;
2518 : char uuid[UUID_MAX];
2519 : uint32_t v_pos;
2520 : uint32_t v_total_blocks;
2521 : uint32_t v_free_blocks;
2522 : struct snapraid_disk* disk;
2523 :
2524 1403 : ret = sgetbs(f, buffer, sizeof(buffer));
2525 1403 : if (ret < 0) {
2526 : /* LCOV_EXCL_START */
2527 : decoding_error(path, f);
2528 : os_abort();
2529 : /* LCOV_EXCL_STOP */
2530 : }
2531 :
2532 1403 : ret = sgetb32(f, &v_pos);
2533 1403 : if (ret < 0) {
2534 : /* LCOV_EXCL_START */
2535 : decoding_error(path, f);
2536 : os_abort();
2537 : /* LCOV_EXCL_STOP */
2538 : }
2539 :
2540 : /* from SnapRAID 7.0 the 'M' command includes the free space */
2541 1403 : if (c == 'M') {
2542 1403 : ret = sgetb32(f, &v_total_blocks);
2543 1403 : if (ret < 0) {
2544 : /* LCOV_EXCL_START */
2545 : decoding_error(path, f);
2546 : os_abort();
2547 : /* LCOV_EXCL_STOP */
2548 : }
2549 :
2550 1403 : ret = sgetb32(f, &v_free_blocks);
2551 1403 : if (ret < 0) {
2552 : /* LCOV_EXCL_START */
2553 : decoding_error(path, f);
2554 : os_abort();
2555 : /* LCOV_EXCL_STOP */
2556 : }
2557 : } else {
2558 0 : v_total_blocks = 0;
2559 0 : v_free_blocks = 0;
2560 : }
2561 :
2562 : /* read the uuid */
2563 1403 : ret = sgetbs(f, uuid, sizeof(uuid));
2564 1403 : if (ret < 0) {
2565 : /* LCOV_EXCL_START */
2566 : decoding_error(path, f);
2567 : os_abort();
2568 : /* LCOV_EXCL_STOP */
2569 : }
2570 :
2571 : /* find the disk */
2572 1403 : disk = find_disk_by_name(state, buffer);
2573 1403 : if (!disk) {
2574 : /* search by UUID if renamed */
2575 2 : disk = find_disk_by_uuid(state, uuid);
2576 2 : if (disk) {
2577 2 : log_fatal("WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2578 :
2579 : /* write the new state with the new name */
2580 2 : state->need_write = 1;
2581 : }
2582 : }
2583 1403 : if (!disk) {
2584 : /* LCOV_EXCL_START */
2585 : decoding_error(path, f);
2586 : log_fatal("Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2587 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2588 : /* if it's a command without UUID, it cannot autorename using UUID */
2589 : if (state->opt.skip_disk_access)
2590 : log_fatal("If you have renamed it, run 'sync' to update the new name\n");
2591 : exit(EXIT_FAILURE);
2592 : /* LCOV_EXCL_STOP */
2593 : }
2594 :
2595 1403 : map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2596 :
2597 1403 : tommy_list_insert_tail(&state->maplist, &map->node, map);
2598 :
2599 : /* insert in the mapping vector */
2600 1403 : tommy_array_grow(&disk_mapping, mapping_max + 1);
2601 1403 : tommy_array_set(&disk_mapping, mapping_max, disk);
2602 1403 : ++mapping_max;
2603 1668 : } else if (c == 'P') {
2604 : /* from SnapRAID 7.0 the 'P' command includes the free space */
2605 : /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2606 : char v_uuid[UUID_MAX];
2607 : uint32_t v_level;
2608 : uint32_t v_total_blocks;
2609 : uint32_t v_free_blocks;
2610 :
2611 0 : ret = sgetb32(f, &v_level);
2612 0 : if (ret < 0) {
2613 : /* LCOV_EXCL_START */
2614 : decoding_error(path, f);
2615 : os_abort();
2616 : /* LCOV_EXCL_STOP */
2617 : }
2618 :
2619 0 : ret = sgetb32(f, &v_total_blocks);
2620 0 : if (ret < 0) {
2621 : /* LCOV_EXCL_START */
2622 : decoding_error(path, f);
2623 : os_abort();
2624 : /* LCOV_EXCL_STOP */
2625 : }
2626 :
2627 0 : ret = sgetb32(f, &v_free_blocks);
2628 0 : if (ret < 0) {
2629 : /* LCOV_EXCL_START */
2630 : decoding_error(path, f);
2631 : os_abort();
2632 : /* LCOV_EXCL_STOP */
2633 : }
2634 :
2635 0 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2636 0 : if (ret < 0) {
2637 : /* LCOV_EXCL_START */
2638 : decoding_error(path, f);
2639 : os_abort();
2640 : /* LCOV_EXCL_STOP */
2641 : }
2642 :
2643 0 : if (v_level >= LEV_MAX) {
2644 : /* LCOV_EXCL_START */
2645 : decoding_error(path, f);
2646 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2647 : exit(EXIT_FAILURE);
2648 : /* LCOV_EXCL_STOP */
2649 : }
2650 :
2651 : /* auto configure if configuration is missing */
2652 0 : if (state->no_conf) {
2653 0 : if (v_level >= state->level)
2654 0 : state->level = v_level + 1;
2655 : }
2656 :
2657 : /* if we use this parity entry */
2658 0 : if (v_level < state->level) {
2659 : /* if the configuration has more splits, keep them */
2660 0 : if (state->parity[v_level].split_mac < 1)
2661 0 : state->parity[v_level].split_mac = 1;
2662 : /* set the parity info */
2663 0 : pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
2664 0 : state->parity[v_level].total_blocks = v_total_blocks;
2665 0 : state->parity[v_level].free_blocks = v_free_blocks;
2666 : }
2667 1668 : } else if (c == 'Q') {
2668 : /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
2669 : uint32_t v_level;
2670 : uint32_t v_total_blocks;
2671 : uint32_t v_free_blocks;
2672 : uint32_t v_split_mac;
2673 : unsigned s;
2674 :
2675 1415 : ret = sgetb32(f, &v_level);
2676 1415 : if (ret < 0) {
2677 : /* LCOV_EXCL_START */
2678 : decoding_error(path, f);
2679 : os_abort();
2680 : /* LCOV_EXCL_STOP */
2681 : }
2682 :
2683 1415 : ret = sgetb32(f, &v_total_blocks);
2684 1415 : if (ret < 0) {
2685 : /* LCOV_EXCL_START */
2686 : decoding_error(path, f);
2687 : os_abort();
2688 : /* LCOV_EXCL_STOP */
2689 : }
2690 :
2691 1415 : ret = sgetb32(f, &v_free_blocks);
2692 1415 : if (ret < 0) {
2693 : /* LCOV_EXCL_START */
2694 : decoding_error(path, f);
2695 : os_abort();
2696 : /* LCOV_EXCL_STOP */
2697 : }
2698 :
2699 1415 : ret = sgetb32(f, &v_split_mac);
2700 1415 : if (ret < 0) {
2701 : /* LCOV_EXCL_START */
2702 : decoding_error(path, f);
2703 : os_abort();
2704 : /* LCOV_EXCL_STOP */
2705 : }
2706 :
2707 1415 : if (v_level >= LEV_MAX) {
2708 : /* LCOV_EXCL_START */
2709 : decoding_error(path, f);
2710 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2711 : exit(EXIT_FAILURE);
2712 : /* LCOV_EXCL_STOP */
2713 : }
2714 :
2715 : /* auto configure if configuration is missing */
2716 1415 : if (state->no_conf) {
2717 6 : if (v_level >= state->level)
2718 5 : state->level = v_level + 1;
2719 6 : if (state->parity[v_level].split_mac < v_split_mac)
2720 6 : state->parity[v_level].split_mac = v_split_mac;
2721 : }
2722 :
2723 : /* if we use this parity entry */
2724 1415 : if (v_level < state->level) {
2725 : /* set the parity info */
2726 1156 : state->parity[v_level].total_blocks = v_total_blocks;
2727 1156 : state->parity[v_level].free_blocks = v_free_blocks;
2728 : }
2729 :
2730 7072 : for (s = 0; s < v_split_mac; ++s) {
2731 : char v_path[PATH_MAX];
2732 : char v_uuid[UUID_MAX];
2733 : uint64_t v_size;
2734 :
2735 5658 : ret = sgetbs(f, v_path, sizeof(v_path));
2736 5658 : if (ret < 0) {
2737 : /* LCOV_EXCL_START */
2738 : decoding_error(path, f);
2739 : os_abort();
2740 : /* LCOV_EXCL_STOP */
2741 : }
2742 :
2743 5657 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2744 5657 : if (ret < 0) {
2745 : /* LCOV_EXCL_START */
2746 : decoding_error(path, f);
2747 : os_abort();
2748 : /* LCOV_EXCL_STOP */
2749 : }
2750 :
2751 5657 : ret = sgetb64(f, &v_size);
2752 5657 : if (ret < 0) {
2753 : /* LCOV_EXCL_START */
2754 : decoding_error(path, f);
2755 : os_abort();
2756 : /* LCOV_EXCL_STOP */
2757 : }
2758 :
2759 : /* if we use this parity entry */
2760 5657 : if (v_level < state->level) {
2761 : /* if this split was removed from the configuration */
2762 4621 : if (s >= state->parity[v_level].split_mac) {
2763 : /* if the file is used, we really need it */
2764 0 : if (v_size != 0) {
2765 : /* LCOV_EXCL_START */
2766 : decoding_error(path, f);
2767 : log_fatal("Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
2768 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2769 : exit(EXIT_FAILURE);
2770 : /* LCOV_EXCL_STOP */
2771 : }
2772 :
2773 : /* otherwise we can drop it */
2774 0 : log_fatal("WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
2775 : } else {
2776 : /* we copy the path only if without configuration file */
2777 4621 : if (state->no_conf)
2778 24 : pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
2779 :
2780 : /* set the split info */
2781 4621 : pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
2782 4621 : state->parity[v_level].split_map[s].size = v_size;
2783 :
2784 : /* log the info read from the content file */
2785 9242 : log_tag("content:%s:%u:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), s,
2786 4621 : state->parity[v_level].split_map[s].path,
2787 4621 : state->parity[v_level].split_map[s].uuid,
2788 : state->parity[v_level].split_map[s].size);
2789 : }
2790 : }
2791 : }
2792 253 : } else if (c == 'N') {
2793 : uint32_t crc_stored;
2794 : uint32_t crc_computed;
2795 :
2796 : /* get the crc before reading it from the file */
2797 253 : crc_computed = scrc(f);
2798 :
2799 253 : ret = sgetble32(f, &crc_stored);
2800 253 : if (ret < 0) {
2801 : /* LCOV_EXCL_START */
2802 : /* here don't call decoding_error() because it's too late to get the crc */
2803 : log_fatal("Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
2804 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2805 : exit(EXIT_FAILURE);
2806 : /* LCOV_EXCL_STOP */
2807 : }
2808 :
2809 253 : if (crc_stored != crc_computed) {
2810 : /* LCOV_EXCL_START */
2811 : /* here don't call decoding_error() because it's too late to get the crc */
2812 : log_fatal("Mismatching CRC in '%s'\n", path);
2813 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2814 : exit(EXIT_FAILURE);
2815 : /* LCOV_EXCL_STOP */
2816 : }
2817 :
2818 253 : crc_checked = 1;
2819 : } else {
2820 : /* LCOV_EXCL_START */
2821 : decoding_error(path, f);
2822 : log_fatal("Invalid command '%c'!\n", (char)c);
2823 : os_abort();
2824 : /* LCOV_EXCL_STOP */
2825 : }
2826 3735357 : }
2827 :
2828 253 : tommy_array_done(&disk_mapping);
2829 :
2830 253 : if (serror(f)) {
2831 : /* LCOV_EXCL_START */
2832 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2833 : exit(EXIT_FAILURE);
2834 : /* LCOV_EXCL_STOP */
2835 : }
2836 :
2837 253 : if (!crc_checked) {
2838 : /* LCOV_EXCL_START */
2839 : log_fatal("Finished reading '%s' without finding the CRC\n", path);
2840 : log_fatal("This content file is truncated or damaged! Use an alternate copy.\n");
2841 : exit(EXIT_FAILURE);
2842 : /* LCOV_EXCL_STOP */
2843 : }
2844 :
2845 : /* check the file-system on all disks */
2846 253 : state_fscheck(state, "after read");
2847 :
2848 : /* check that the stored parity size matches the loaded state */
2849 253 : if (blockmax != parity_allocated_size(state)) {
2850 : /* LCOV_EXCL_START */
2851 : log_fatal("Internal inconsistency in parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
2852 : if (state->opt.skip_content_check) {
2853 : log_fatal("Overriding.\n");
2854 : blockmax = parity_allocated_size(state);
2855 : } else {
2856 : exit(EXIT_FAILURE);
2857 : }
2858 : /* LCOV_EXCL_STOP */
2859 : }
2860 :
2861 253 : msg_verbose("%8u files\n", count_file);
2862 253 : msg_verbose("%8u hardlinks\n", count_hardlink);
2863 253 : msg_verbose("%8u symlinks\n", count_symlink);
2864 253 : msg_verbose("%8u empty dirs\n", count_dir);
2865 253 : }
2866 :
2867 : struct state_write_thread_context {
2868 : struct snapraid_state* state;
2869 : #if HAVE_MT_WRITE
2870 : pthread_t thread;
2871 : #endif
2872 : /* input */
2873 : block_off_t blockmax;
2874 : time_t info_oldest;
2875 : int info_has_rehash;
2876 : STREAM* f;
2877 : /* output */
2878 : uint32_t crc;
2879 : unsigned count_file;
2880 : unsigned count_hardlink;
2881 : unsigned count_symlink;
2882 : unsigned count_dir;
2883 : };
2884 :
2885 151 : static void* state_write_thread(void* arg)
2886 : {
2887 151 : struct state_write_thread_context* context = arg;
2888 151 : struct snapraid_state* state = context->state;
2889 151 : block_off_t blockmax = context->blockmax;
2890 151 : time_t info_oldest = context->info_oldest;
2891 151 : int info_has_rehash = context->info_has_rehash;
2892 151 : STREAM* f = context->f;
2893 : uint32_t crc;
2894 : unsigned count_file;
2895 : unsigned count_hardlink;
2896 : unsigned count_symlink;
2897 : unsigned count_dir;
2898 : tommy_node* i;
2899 : block_off_t idx;
2900 : block_off_t begin;
2901 : unsigned l, s;
2902 : int version;
2903 :
2904 151 : count_file = 0;
2905 151 : count_hardlink = 0;
2906 151 : count_symlink = 0;
2907 151 : count_dir = 0;
2908 :
2909 : /* check what version to use */
2910 151 : version = 2;
2911 1012 : for (l = 0; l < state->level; ++l) {
2912 861 : if (state->parity[l].split_mac > 1)
2913 861 : version = 3;
2914 : }
2915 151 : if (BLOCK_HASH_SIZE != 16)
2916 0 : version = 3;
2917 :
2918 : /* write header */
2919 151 : if (version == 3)
2920 151 : swrite("SNAPCNT3\n\3\0\0", 12, f);
2921 : else
2922 0 : swrite("SNAPCNT2\n\3\0\0", 12, f);
2923 :
2924 : /* write block size and block max */
2925 151 : sputc('z', f);
2926 151 : sputb32(state->block_size, f);
2927 151 : sputc('x', f);
2928 151 : sputb32(blockmax, f);
2929 :
2930 : /* hash size */
2931 151 : if (version == 3) {
2932 151 : sputc('y', f);
2933 151 : sputb32(BLOCK_HASH_SIZE, f);
2934 : }
2935 :
2936 151 : if (serror(f)) {
2937 : /* LCOV_EXCL_START */
2938 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2939 : return context;
2940 : /* LCOV_EXCL_STOP */
2941 : }
2942 :
2943 151 : sputc('c', f);
2944 151 : if (state->hash == HASH_MURMUR3) {
2945 96 : sputc('u', f);
2946 55 : } else if (state->hash == HASH_SPOOKY2) {
2947 55 : sputc('k', f);
2948 : } else {
2949 : /* LCOV_EXCL_START */
2950 : log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
2951 : return context;
2952 : /* LCOV_EXCL_STOP */
2953 : }
2954 151 : swrite(state->hashseed, HASH_MAX, f);
2955 151 : if (serror(f)) {
2956 : /* LCOV_EXCL_START */
2957 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2958 : return context;
2959 : /* LCOV_EXCL_STOP */
2960 : }
2961 :
2962 : /* previous hash only present */
2963 151 : if (state->prevhash != HASH_UNDEFINED) {
2964 : /* if at least one rehash tag found, we have to save the previous hash */
2965 3 : if (info_has_rehash) {
2966 2 : sputc('C', f);
2967 2 : if (state->prevhash == HASH_MURMUR3) {
2968 2 : sputc('u', f);
2969 0 : } else if (state->prevhash == HASH_SPOOKY2) {
2970 0 : sputc('k', f);
2971 : } else {
2972 : /* LCOV_EXCL_START */
2973 : log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
2974 : return context;
2975 : /* LCOV_EXCL_STOP */
2976 : }
2977 2 : swrite(state->prevhashseed, HASH_MAX, f);
2978 2 : if (serror(f)) {
2979 : /* LCOV_EXCL_START */
2980 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
2981 : return context;
2982 : /* LCOV_EXCL_STOP */
2983 : }
2984 : }
2985 : }
2986 :
2987 : /* for each map */
2988 1057 : for (i = state->maplist; i != 0; i = i->next) {
2989 906 : struct snapraid_map* map = i->data;
2990 : struct snapraid_disk* disk;
2991 :
2992 : /* find the disk for this mapping */
2993 906 : disk = find_disk_by_name(state, map->name);
2994 906 : if (!disk) {
2995 : /* LCOV_EXCL_START */
2996 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
2997 : return context;
2998 : /* LCOV_EXCL_STOP */
2999 : }
3000 :
3001 : /* save the mapping only if disk is mapped */
3002 906 : if (disk->mapping_idx != -1) {
3003 814 : sputc('M', f);
3004 814 : sputbs(map->name, f);
3005 814 : sputb32(map->position, f);
3006 814 : sputb32(map->total_blocks, f);
3007 814 : sputb32(map->free_blocks, f);
3008 814 : sputbs(map->uuid, f);
3009 814 : if (serror(f)) {
3010 : /* LCOV_EXCL_START */
3011 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3012 : return context;
3013 : /* LCOV_EXCL_STOP */
3014 : }
3015 : }
3016 : }
3017 :
3018 : /* for each parity */
3019 1012 : for (l = 0; l < state->level; ++l) {
3020 861 : if (version == 3) {
3021 861 : sputc('Q', f);
3022 861 : sputb32(l, f);
3023 861 : sputb32(state->parity[l].total_blocks, f);
3024 861 : sputb32(state->parity[l].free_blocks, f);
3025 861 : sputb32(state->parity[l].split_mac, f);
3026 4305 : for (s = 0; s < state->parity[l].split_mac; ++s) {
3027 3444 : sputbs(state->parity[l].split_map[s].path, f);
3028 3444 : sputbs(state->parity[l].split_map[s].uuid, f);
3029 3444 : sputb64(state->parity[l].split_map[s].size, f);
3030 : }
3031 : } else {
3032 0 : sputc('P', f);
3033 0 : sputb32(l, f);
3034 0 : sputb32(state->parity[l].total_blocks, f);
3035 0 : sputb32(state->parity[l].free_blocks, f);
3036 0 : sputbs(state->parity[l].split_map[0].uuid, f);
3037 : }
3038 861 : if (serror(f)) {
3039 : /* LCOV_EXCL_START */
3040 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3041 : return context;
3042 : /* LCOV_EXCL_STOP */
3043 : }
3044 : }
3045 :
3046 : /* for each disk */
3047 1057 : for (i = state->disklist; i != 0; i = i->next) {
3048 : tommy_node* j;
3049 906 : struct snapraid_disk* disk = i->data;
3050 :
3051 : /* if the disk is not mapped, skip it */
3052 906 : if (disk->mapping_idx < 0)
3053 92 : continue;
3054 :
3055 : /* for each file */
3056 1771698 : for (j = disk->filelist; j != 0; j = j->next) {
3057 1770884 : struct snapraid_file* file = j->data;
3058 : uint64_t size;
3059 : uint64_t mtime_sec;
3060 : int32_t mtime_nsec;
3061 : uint64_t inode;
3062 :
3063 1770884 : size = file->size;
3064 1770884 : mtime_sec = file->mtime_sec;
3065 1770884 : mtime_nsec = file->mtime_nsec;
3066 1770884 : inode = file->inode;
3067 :
3068 1770884 : sputc('f', f);
3069 1770884 : sputb32(disk->mapping_idx, f);
3070 1770884 : sputb64(size, f);
3071 1770884 : sputb64(mtime_sec, f);
3072 : /* encode STAT_NSEC_INVALID as 0 */
3073 1770884 : if (mtime_nsec == STAT_NSEC_INVALID)
3074 0 : sputb32(0, f);
3075 : else
3076 1770884 : sputb32(mtime_nsec + 1, f);
3077 1770884 : sputb64(inode, f);
3078 1770884 : sputbs(file->sub, f);
3079 1770884 : if (serror(f)) {
3080 : /* LCOV_EXCL_START */
3081 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3082 : return context;
3083 : /* LCOV_EXCL_STOP */
3084 : }
3085 :
3086 : /* for all the blocks of the file */
3087 1770884 : begin = 0;
3088 5322957 : while (begin < file->blockmax) {
3089 1781189 : unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3090 1781189 : block_off_t v_pos = fs_file2par_get(disk, file, begin);
3091 : uint32_t v_count;
3092 :
3093 : block_off_t end;
3094 :
3095 : /* find the end of run of blocks */
3096 1781189 : end = begin + 1;
3097 6146329 : while (end < file->blockmax) {
3098 2596728 : if (v_state != block_state_get(fs_file2block_get(file, end)))
3099 12 : break;
3100 2596716 : if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3101 12765 : break;
3102 2583951 : ++end;
3103 : }
3104 :
3105 1781189 : switch (v_state) {
3106 : case BLOCK_STATE_BLK :
3107 1678557 : sputc('b', f);
3108 1678557 : break;
3109 : case BLOCK_STATE_CHG :
3110 67714 : sputc('g', f);
3111 67714 : break;
3112 : case BLOCK_STATE_REP :
3113 34918 : sputc('p', f);
3114 34918 : break;
3115 : default :
3116 : /* LCOV_EXCL_START */
3117 : log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, v_state);
3118 : return context;
3119 : /* LCOV_EXCL_STOP */
3120 : }
3121 :
3122 1781189 : sputb32(v_pos, f);
3123 :
3124 1781189 : v_count = end - begin;
3125 1781189 : sputb32(v_count, f);
3126 :
3127 : /* write hashes */
3128 6146329 : for (idx = begin; idx < end; ++idx) {
3129 4365140 : struct snapraid_block* block = fs_file2block_get(file, idx);
3130 :
3131 4365140 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3132 : }
3133 :
3134 1781189 : if (serror(f)) {
3135 : /* LCOV_EXCL_START */
3136 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3137 : return context;
3138 : /* LCOV_EXCL_STOP */
3139 : }
3140 :
3141 : /* next begin position */
3142 1781189 : begin = end;
3143 : }
3144 :
3145 1770884 : ++count_file;
3146 : }
3147 :
3148 : /* for each link */
3149 58887 : for (j = disk->linklist; j != 0; j = j->next) {
3150 58073 : struct snapraid_link* slink = j->data;
3151 :
3152 58073 : switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3153 : case FILE_IS_HARDLINK :
3154 321 : sputc('a', f);
3155 321 : ++count_hardlink;
3156 321 : break;
3157 : case FILE_IS_SYMLINK :
3158 57752 : sputc('s', f);
3159 57752 : ++count_symlink;
3160 57752 : break;
3161 : }
3162 :
3163 58073 : sputb32(disk->mapping_idx, f);
3164 58073 : sputbs(slink->sub, f);
3165 58073 : sputbs(slink->linkto, f);
3166 58073 : if (serror(f)) {
3167 : /* LCOV_EXCL_START */
3168 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3169 : return context;
3170 : /* LCOV_EXCL_STOP */
3171 : }
3172 : }
3173 :
3174 : /* for each dir */
3175 1241 : for (j = disk->dirlist; j != 0; j = j->next) {
3176 427 : struct snapraid_dir* dir = j->data;
3177 :
3178 427 : sputc('r', f);
3179 427 : sputb32(disk->mapping_idx, f);
3180 427 : sputbs(dir->sub, f);
3181 427 : if (serror(f)) {
3182 : /* LCOV_EXCL_START */
3183 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3184 : return context;
3185 : /* LCOV_EXCL_STOP */
3186 : }
3187 :
3188 427 : ++count_dir;
3189 : }
3190 :
3191 : /* deleted blocks of the disk */
3192 814 : sputc('h', f);
3193 814 : sputb32(disk->mapping_idx, f);
3194 814 : if (serror(f)) {
3195 : /* LCOV_EXCL_START */
3196 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3197 : return context;
3198 : /* LCOV_EXCL_STOP */
3199 : }
3200 814 : begin = 0;
3201 2516 : while (begin < blockmax) {
3202 : int is_deleted;
3203 : block_off_t end;
3204 :
3205 888 : is_deleted = fs_is_block_deleted(disk, begin);
3206 :
3207 : /* find the end of run of blocks */
3208 888 : end = begin + 1;
3209 4530435 : while (end < blockmax
3210 4528733 : && is_deleted == fs_is_block_deleted(disk, end)
3211 : ) {
3212 4528659 : ++end;
3213 : }
3214 :
3215 888 : sputb32(end - begin, f);
3216 :
3217 888 : if (is_deleted) {
3218 : /* write the run of deleted blocks with hash */
3219 47 : sputc('o', f);
3220 :
3221 : /* write all the hash */
3222 15961 : while (begin < end) {
3223 15867 : struct snapraid_block* block = fs_par2block_get(disk, begin);
3224 :
3225 15867 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3226 :
3227 15867 : ++begin;
3228 : }
3229 : } else {
3230 : /* write the run of blocks without hash */
3231 : /* they can be either used or empty blocks */
3232 841 : sputc('O', f);
3233 :
3234 : /* next begin position */
3235 841 : begin = end;
3236 : }
3237 :
3238 888 : if (serror(f)) {
3239 : /* LCOV_EXCL_START */
3240 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3241 : return context;
3242 : /* LCOV_EXCL_STOP */
3243 : }
3244 : }
3245 : }
3246 :
3247 : /* write the info for each block */
3248 151 : sputc('i', f);
3249 151 : sputb32(info_oldest, f);
3250 151 : begin = 0;
3251 19663 : while (begin < blockmax) {
3252 : snapraid_info info;
3253 : block_off_t end;
3254 : time_t t;
3255 : unsigned flag;
3256 :
3257 19361 : info = info_get(&state->infoarr, begin);
3258 :
3259 : /* find the end of run of blocks */
3260 19361 : end = begin + 1;
3261 775460 : while (end < blockmax
3262 755951 : && info == info_get(&state->infoarr, end)
3263 : ) {
3264 736738 : ++end;
3265 : }
3266 :
3267 19361 : sputb32(end - begin, f);
3268 :
3269 : /* if there is info */
3270 19361 : if (info) {
3271 : /* other flags */
3272 19338 : flag = 1; /* info is present */
3273 19338 : if (info_get_bad(info))
3274 1331 : flag |= 2;
3275 19338 : if (info_get_rehash(info))
3276 5212 : flag |= 4;
3277 19338 : if (info_get_justsynced(info))
3278 12001 : flag |= 8;
3279 19338 : sputb32(flag, f);
3280 :
3281 19338 : t = info_get_time(info) - info_oldest;
3282 19338 : sputb32(t, f);
3283 : } else {
3284 : /* write a special 0 flag to mark missing info */
3285 23 : sputb32(0, f);
3286 : }
3287 :
3288 19361 : if (serror(f)) {
3289 : /* LCOV_EXCL_START */
3290 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3291 : return context;
3292 : /* LCOV_EXCL_STOP */
3293 : }
3294 :
3295 : /* next begin position */
3296 19361 : begin = end;
3297 : }
3298 :
3299 151 : sputc('N', f);
3300 :
3301 : /* flush data written to the disk */
3302 151 : if (sflush(f)) {
3303 : /* LCOV_EXCL_START */
3304 : log_fatal("Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3305 : return context;
3306 : /* LCOV_EXCL_STOP */
3307 : }
3308 :
3309 : /* get the file crc */
3310 151 : crc = scrc(f);
3311 :
3312 : /* compare the crc of the data written to file */
3313 : /* with the one of the data written to the stream */
3314 151 : if (crc != scrc_stream(f)) {
3315 : /* LCOV_EXCL_START */
3316 : log_fatal("CRC mismatch writing the content stream.\n");
3317 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3318 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3319 : return context;
3320 : /* LCOV_EXCL_STOP */
3321 : }
3322 :
3323 151 : sputble32(crc, f);
3324 151 : if (serror(f)) {
3325 : /* LCOV_EXCL_START */
3326 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3327 : return context;
3328 : /* LCOV_EXCL_STOP */
3329 : }
3330 :
3331 : /* set output variables */
3332 151 : context->crc = crc;
3333 151 : context->count_file = count_file;
3334 151 : context->count_hardlink = count_hardlink;
3335 151 : context->count_symlink = count_symlink;
3336 151 : context->count_dir = count_dir;
3337 :
3338 151 : return 0;
3339 : }
3340 :
3341 151 : static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3342 : {
3343 : #if HAVE_MT_WRITE
3344 : int fail;
3345 : int first;
3346 : #else
3347 : STREAM* f;
3348 : unsigned count_content;
3349 : unsigned k;
3350 : struct state_write_thread_context* context;
3351 : void* retval;
3352 : #endif
3353 : tommy_node* i;
3354 : block_off_t blockmax;
3355 : time_t info_oldest;
3356 : int info_has_rehash;
3357 : int mapping_idx;
3358 : block_off_t idx;
3359 : uint32_t crc;
3360 : unsigned count_file;
3361 : unsigned count_hardlink;
3362 : unsigned count_symlink;
3363 : unsigned count_dir;
3364 :
3365 : /* blocks of all array */
3366 151 : blockmax = parity_allocated_size(state);
3367 :
3368 : /* check the file-system on all disks */
3369 151 : state_fscheck(state, "before write");
3370 :
3371 : /* clear the info for unused blocks */
3372 : /* and get some other info */
3373 151 : info_oldest = 0; /* oldest time in info */
3374 151 : info_has_rehash = 0; /* if there is a rehash info */
3375 756250 : for (idx = 0; idx < blockmax; ++idx) {
3376 : /* if the position is used */
3377 756099 : if (fs_position_is_required(state, idx)) {
3378 756099 : snapraid_info info = info_get(&state->infoarr, idx);
3379 :
3380 : /* only if there is some info to store */
3381 756099 : if (info) {
3382 711614 : time_t info_time = info_get_time(info);
3383 :
3384 711614 : if (!info_oldest || info_time < info_oldest)
3385 147 : info_oldest = info_time;
3386 :
3387 711614 : if (info_get_rehash(info))
3388 13822 : info_has_rehash = 1;
3389 : }
3390 : } else {
3391 : /* clear any previous info */
3392 0 : info_set(&state->infoarr, idx, 0);
3393 :
3394 : /* and clear any deleted blocks */
3395 0 : fs_position_clear_deleted(state, idx);
3396 : }
3397 : }
3398 :
3399 : /* map disks */
3400 151 : mapping_idx = 0;
3401 1057 : for (i = state->maplist; i != 0; i = i->next) {
3402 906 : struct snapraid_map* map = i->data;
3403 : struct snapraid_disk* disk;
3404 :
3405 : /* find the disk for this mapping */
3406 906 : disk = find_disk_by_name(state, map->name);
3407 906 : if (!disk) {
3408 : /* LCOV_EXCL_START */
3409 : log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name);
3410 : os_abort();
3411 : /* LCOV_EXCL_STOP */
3412 : }
3413 :
3414 : /* save the mapping only for not empty disks */
3415 906 : if (!fs_is_empty(disk, blockmax)) {
3416 : /* assign the mapping index used to identify disks */
3417 814 : disk->mapping_idx = mapping_idx;
3418 814 : ++mapping_idx;
3419 : } else {
3420 : /* mark the disk as without mapping */
3421 92 : disk->mapping_idx = -1;
3422 : }
3423 : }
3424 :
3425 : #if HAVE_MT_WRITE
3426 : /* start all writing threads */
3427 : i = tommy_list_head(&state->contentlist);
3428 : while (i) {
3429 : struct snapraid_content* content = i->data;
3430 : struct state_write_thread_context* context;
3431 : char tmp[PATH_MAX];
3432 : STREAM* f;
3433 :
3434 : msg_progress("Saving state to %s...\n", content->content);
3435 :
3436 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3437 : f = sopen_write(tmp);
3438 : if (f == 0) {
3439 : /* LCOV_EXCL_START */
3440 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3441 : exit(EXIT_FAILURE);
3442 : /* LCOV_EXCL_STOP */
3443 : }
3444 :
3445 : /* allocate the thread context */
3446 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3447 : content->context = context;
3448 :
3449 : /* initialize */
3450 : context->state = state;
3451 : context->blockmax = blockmax;
3452 : context->info_oldest = info_oldest;
3453 : context->info_has_rehash = info_has_rehash;
3454 : context->f = f;
3455 :
3456 : thread_create(&context->thread, 0, state_write_thread, context);
3457 :
3458 : i = i->next;
3459 : }
3460 :
3461 : /* join all thread */
3462 : fail = 0;
3463 : first = 1;
3464 : crc = 0;
3465 : count_file = 0;
3466 : count_hardlink = 0;
3467 : count_symlink = 0;
3468 : count_dir = 0;
3469 : i = tommy_list_head(&state->contentlist);
3470 : while (i) {
3471 : struct snapraid_content* content = i->data;
3472 : struct state_write_thread_context* context = content->context;
3473 : void* retval;
3474 :
3475 : thread_join(context->thread, &retval);
3476 :
3477 : if (retval) {
3478 : /* LCOV_EXCL_START */
3479 : fail = 1;
3480 : /* LCOV_EXCL_STOP */
3481 : } else {
3482 : STREAM* f = context->f;
3483 :
3484 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3485 : /* than even in a system crash event we have one valid copy of the file. */
3486 : if (sflush(f) != 0) {
3487 : /* LCOV_EXCL_START */
3488 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3489 : exit(EXIT_FAILURE);
3490 : /* LCOV_EXCL_STOP */
3491 : }
3492 :
3493 : #if HAVE_FSYNC
3494 : if (ssync(f) != 0) {
3495 : /* LCOV_EXCL_START */
3496 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3497 : exit(EXIT_FAILURE);
3498 : /* LCOV_EXCL_STOP */
3499 : }
3500 : #endif
3501 :
3502 : if (sclose(f) != 0) {
3503 : /* LCOV_EXCL_START */
3504 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3505 : exit(EXIT_FAILURE);
3506 : /* LCOV_EXCL_STOP */
3507 : }
3508 :
3509 : if (first) {
3510 : first = 0;
3511 : crc = context->crc;
3512 : count_file = context->count_file;
3513 : count_hardlink = context->count_hardlink;
3514 : count_symlink = context->count_symlink;
3515 : count_dir = context->count_dir;
3516 : } else {
3517 : if (crc != context->crc) {
3518 : /* LCOV_EXCL_START */
3519 : log_fatal("Different CRCs writing content streams.\n");
3520 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3521 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3522 : exit(EXIT_FAILURE);
3523 : /* LCOV_EXCL_STOP */
3524 : }
3525 : }
3526 : }
3527 :
3528 : free(context);
3529 :
3530 : i = i->next;
3531 : }
3532 :
3533 : /* abort on failure */
3534 : if (fail) {
3535 : /* LCOV_EXCL_START */
3536 : exit(EXIT_FAILURE);
3537 : /* LCOV_EXCL_STOP */
3538 : }
3539 : #else
3540 : /* count the content files */
3541 151 : count_content = 0;
3542 151 : i = tommy_list_head(&state->contentlist);
3543 1314 : while (i) {
3544 1012 : struct snapraid_content* content = i->data;
3545 1012 : msg_progress("Saving state to %s...\n", content->content);
3546 1012 : ++count_content;
3547 1012 : i = i->next;
3548 : }
3549 :
3550 : /* open all the content files */
3551 151 : f = sopen_multi_write(count_content);
3552 151 : if (!f) {
3553 : /* LCOV_EXCL_START */
3554 : log_fatal("Error opening the content files.\n");
3555 : exit(EXIT_FAILURE);
3556 : /* LCOV_EXCL_STOP */
3557 : }
3558 :
3559 151 : k = 0;
3560 151 : i = tommy_list_head(&state->contentlist);
3561 1314 : while (i) {
3562 1012 : struct snapraid_content* content = i->data;
3563 : char tmp[PATH_MAX];
3564 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3565 1012 : if (sopen_multi_file(f, k, tmp) != 0) {
3566 : /* LCOV_EXCL_START */
3567 : log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
3568 : exit(EXIT_FAILURE);
3569 : /* LCOV_EXCL_STOP */
3570 : }
3571 1012 : ++k;
3572 1012 : i = i->next;
3573 : }
3574 :
3575 : /* allocate the thread context */
3576 151 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3577 :
3578 : /* initialize */
3579 151 : context->state = state;
3580 151 : context->blockmax = blockmax;
3581 151 : context->info_oldest = info_oldest;
3582 151 : context->info_has_rehash = info_has_rehash;
3583 151 : context->f = f;
3584 :
3585 151 : retval = state_write_thread(context);
3586 :
3587 : /* abort on failure */
3588 151 : if (retval) {
3589 : /* LCOV_EXCL_START */
3590 : exit(EXIT_FAILURE);
3591 : /* LCOV_EXCL_STOP */
3592 : }
3593 :
3594 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3595 : /* than even in a system crash event we have one valid copy of the file. */
3596 151 : if (sflush(f) != 0) {
3597 : /* LCOV_EXCL_START */
3598 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3599 : exit(EXIT_FAILURE);
3600 : /* LCOV_EXCL_STOP */
3601 : }
3602 :
3603 : #if HAVE_FSYNC
3604 151 : if (ssync(f) != 0) {
3605 : /* LCOV_EXCL_START */
3606 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3607 : exit(EXIT_FAILURE);
3608 : /* LCOV_EXCL_STOP */
3609 : }
3610 : #endif
3611 :
3612 151 : if (sclose(f) != 0) {
3613 : /* LCOV_EXCL_START */
3614 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3615 : exit(EXIT_FAILURE);
3616 : /* LCOV_EXCL_STOP */
3617 : }
3618 :
3619 151 : crc = context->crc;
3620 151 : count_file = context->count_file;
3621 151 : count_hardlink = context->count_hardlink;
3622 151 : count_symlink = context->count_symlink;
3623 151 : count_dir = context->count_dir;
3624 :
3625 151 : free(context);
3626 : #endif
3627 :
3628 151 : msg_verbose("%8u files\n", count_file);
3629 151 : msg_verbose("%8u hardlinks\n", count_hardlink);
3630 151 : msg_verbose("%8u symlinks\n", count_symlink);
3631 151 : msg_verbose("%8u empty dirs\n", count_dir);
3632 :
3633 151 : *out_crc = crc;
3634 151 : }
3635 :
3636 261 : void state_read(struct snapraid_state* state)
3637 : {
3638 : STREAM* f;
3639 : char path[PATH_MAX];
3640 : struct stat st;
3641 : tommy_node* node;
3642 : int ret;
3643 : int c;
3644 :
3645 : /* iterate over all the available content files and load the first one present */
3646 261 : f = 0;
3647 261 : node = tommy_list_head(&state->contentlist);
3648 572 : while (node) {
3649 304 : struct snapraid_content* content = node->data;
3650 304 : pathcpy(path, sizeof(path), content->content);
3651 :
3652 304 : if (!state->no_conf) {
3653 303 : log_tag("content:%s\n", path);
3654 303 : log_flush();
3655 : }
3656 304 : msg_progress("Loading state from %s...\n", path);
3657 :
3658 304 : f = sopen_read(path);
3659 304 : if (f != 0) {
3660 : /* if opened stop the search */
3661 254 : break;
3662 : } else {
3663 : /* if it's real error of an existing file, abort */
3664 50 : if (errno != ENOENT) {
3665 : /* LCOV_EXCL_START */
3666 : log_fatal("Error opening the content file '%s'. %s.\n", path, strerror(errno));
3667 : exit(EXIT_FAILURE);
3668 : /* LCOV_EXCL_STOP */
3669 : }
3670 :
3671 : /* otherwise continue */
3672 50 : if (node->next) {
3673 43 : log_fatal("WARNING! Content file '%s' not found, trying with another copy...\n", path);
3674 :
3675 : /* ensure to rewrite all the content files */
3676 43 : state->need_write = 1;
3677 : }
3678 : }
3679 :
3680 : /* next content file */
3681 50 : node = node->next;
3682 : }
3683 :
3684 : /* if not found, assume empty */
3685 261 : if (!f) {
3686 7 : log_fatal("No content file found. Assuming empty.\n");
3687 :
3688 : /* create the initial mapping */
3689 7 : state_map(state);
3690 7 : return;
3691 : }
3692 :
3693 : /* get the stat of the content file */
3694 254 : ret = fstat(shandle(f), &st);
3695 254 : if (ret != 0) {
3696 : /* LCOV_EXCL_START */
3697 : log_fatal("Error stating the content file '%s'. %s.\n", path, strerror(errno));
3698 : exit(EXIT_FAILURE);
3699 : /* LCOV_EXCL_STOP */
3700 : }
3701 :
3702 : /* go further to check other content files */
3703 1955 : while (node) {
3704 : char other_path[PATH_MAX];
3705 : struct stat other_st;
3706 1447 : struct snapraid_content* content = node->data;
3707 1447 : pathcpy(other_path, sizeof(other_path), content->content);
3708 :
3709 1447 : ret = stat(other_path, &other_st);
3710 1447 : if (ret != 0) {
3711 : /* allow missing content files, but not any other kind of error */
3712 1 : if (errno != ENOENT) {
3713 : /* LCOV_EXCL_START */
3714 : log_fatal("Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
3715 : exit(EXIT_FAILURE);
3716 : /* LCOV_EXCL_STOP */
3717 : }
3718 :
3719 : /* ensure to rewrite all the content files */
3720 1 : state->need_write = 1;
3721 : } else {
3722 : /* if the size is different */
3723 1446 : if (other_st.st_size != st.st_size) {
3724 39 : log_fatal("WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
3725 39 : log_fatal("Likely one of the two is broken!\n");
3726 :
3727 : /* ensure to rewrite all the content files */
3728 39 : state->need_write = 1;
3729 : }
3730 : }
3731 :
3732 : /* next content file */
3733 1447 : node = node->next;
3734 : }
3735 :
3736 : /* start with a undefined default. */
3737 : /* it's for compatibility with version 1.0 where MD5 was implicit. */
3738 254 : state->hash = HASH_UNDEFINED;
3739 :
3740 : /* start with a zero seed, it was the default in old versions */
3741 254 : memset(state->hashseed, 0, HASH_MAX);
3742 :
3743 : /* previous hash, start with an undefined value */
3744 254 : state->prevhash = HASH_UNDEFINED;
3745 :
3746 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
3747 :
3748 : /* get the first char to detect the file type */
3749 254 : c = sgetc(f);
3750 254 : sungetc(c, f);
3751 :
3752 : /* guess the file type from the first char */
3753 254 : if (c == 'S') {
3754 254 : state_read_content(state, path, f);
3755 : } else {
3756 : /* LCOV_EXCL_START */
3757 : log_fatal("From SnapRAID v9.0 the text content file is not supported anymore.\n");
3758 : log_fatal("You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
3759 : exit(EXIT_FAILURE);
3760 : /* LCOV_EXCL_STOP */
3761 : }
3762 :
3763 253 : sclose(f);
3764 :
3765 253 : if (state->hash == HASH_UNDEFINED) {
3766 : /* LCOV_EXCL_START */
3767 : log_fatal("The checksum to use is not specified.\n");
3768 : log_fatal("This happens because you are likely upgrading from SnapRAID 1.0.\n");
3769 : log_fatal("To use a new SnapRAID you must restart from scratch,\n");
3770 : log_fatal("deleting all the content and parity files.\n");
3771 : exit(EXIT_FAILURE);
3772 : /* LCOV_EXCL_STOP */
3773 : }
3774 :
3775 : /* update the mapping */
3776 253 : state_map(state);
3777 :
3778 253 : state_content_check(state, path);
3779 :
3780 : /* mark that we read the content file, and it passed all the checks */
3781 253 : state->checked_read = 1;
3782 : }
3783 :
3784 : struct state_verify_thread_context {
3785 : struct snapraid_state* state;
3786 : struct snapraid_content* content;
3787 : #if HAVE_MT_VERIFY
3788 : pthread_t thread;
3789 : #else
3790 : void* retval;
3791 : #endif
3792 : /* input */
3793 : uint32_t crc;
3794 : STREAM* f;
3795 : };
3796 :
3797 1009 : static void* state_verify_thread(void* arg)
3798 : {
3799 1009 : struct state_verify_thread_context* context = arg;
3800 1009 : STREAM* f = context->f;
3801 : unsigned char buf[4];
3802 : uint32_t crc_stored;
3803 : uint32_t crc_computed;
3804 :
3805 1009 : if (sdeplete(f, buf) != 0) {
3806 : /* LCOV_EXCL_START */
3807 : log_fatal("Error flushing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3808 : return context;
3809 : /* LCOV_EXCL_STOP */
3810 : }
3811 :
3812 : /* get the stored crc from the last four bytes */
3813 1012 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
3814 :
3815 1012 : if (crc_stored != context->crc) {
3816 : /* LCOV_EXCL_START */
3817 : log_fatal("DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
3818 : return context;
3819 : /* LCOV_EXCL_STOP */
3820 : }
3821 :
3822 : /* get the computed crc */
3823 1012 : crc_computed = scrc(f);
3824 :
3825 : /* adjust the stored crc to include itself */
3826 1012 : crc_stored = crc32c(crc_stored, buf, 4);
3827 :
3828 1012 : if (crc_computed != crc_stored) {
3829 : /* LCOV_EXCL_START */
3830 : log_fatal("DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
3831 : return context;
3832 : /* LCOV_EXCL_STOP */
3833 : }
3834 :
3835 1012 : return 0;
3836 : }
3837 :
3838 151 : static void state_verify_content(struct snapraid_state* state, uint32_t crc)
3839 : {
3840 : tommy_node* i;
3841 : int fail;
3842 :
3843 : /* start all reading threads */
3844 151 : i = tommy_list_head(&state->contentlist);
3845 1314 : while (i) {
3846 1012 : struct snapraid_content* content = i->data;
3847 : struct state_verify_thread_context* context;
3848 : char tmp[PATH_MAX];
3849 : STREAM* f;
3850 :
3851 1012 : msg_progress("Verifying %s...\n", content->content);
3852 :
3853 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3854 :
3855 1012 : f = sopen_read(tmp);
3856 1012 : if (f == 0) {
3857 : /* LCOV_EXCL_START */
3858 : log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno));
3859 : exit(EXIT_FAILURE);
3860 : /* LCOV_EXCL_STOP */
3861 : }
3862 :
3863 : /* allocate the thread context */
3864 1012 : context = malloc_nofail(sizeof(struct state_verify_thread_context));
3865 1012 : content->context = context;
3866 :
3867 : /* initialize */
3868 1012 : context->state = state;
3869 1012 : context->content = content;
3870 1012 : context->crc = crc;
3871 1012 : context->f = f;
3872 :
3873 : #if HAVE_MT_VERIFY
3874 1012 : thread_create(&context->thread, 0, state_verify_thread, context);
3875 : #else
3876 : context->retval = state_verify_thread(context);
3877 : #endif
3878 :
3879 1012 : i = i->next;
3880 : }
3881 :
3882 : /* join all thread */
3883 151 : fail = 0;
3884 151 : i = tommy_list_head(&state->contentlist);
3885 1314 : while (i) {
3886 1012 : struct snapraid_content* content = i->data;
3887 1012 : struct state_verify_thread_context* context = content->context;
3888 : void* retval;
3889 :
3890 : #if HAVE_MT_VERIFY
3891 1012 : thread_join(context->thread, &retval);
3892 : #else
3893 : retval = context->retval;
3894 : #endif
3895 1012 : if (retval) {
3896 : /* LCOV_EXCL_START */
3897 : fail = 1;
3898 : /* LCOV_EXCL_STOP */
3899 : } else {
3900 1012 : STREAM* f = context->f;
3901 :
3902 1012 : if (sclose(f) != 0) {
3903 : /* LCOV_EXCL_START */
3904 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3905 : exit(EXIT_FAILURE);
3906 : /* LCOV_EXCL_STOP */
3907 : }
3908 : }
3909 :
3910 1012 : free(context);
3911 :
3912 1012 : i = i->next;
3913 : }
3914 :
3915 : /* abort on failure */
3916 151 : if (fail) {
3917 : /* LCOV_EXCL_START */
3918 : exit(EXIT_FAILURE);
3919 : /* LCOV_EXCL_STOP */
3920 : }
3921 151 : }
3922 :
3923 151 : static void state_rename_content(struct snapraid_state* state)
3924 : {
3925 : tommy_node* i;
3926 :
3927 151 : i = tommy_list_head(&state->contentlist);
3928 1314 : while (i) {
3929 1012 : struct snapraid_content* content = i->data;
3930 : char tmp[PATH_MAX];
3931 :
3932 : /* now renames the just written copy with the correct name */
3933 1012 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3934 1012 : if (rename(tmp, content->content) != 0) {
3935 : /* LCOV_EXCL_START */
3936 : log_fatal("Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
3937 : exit(EXIT_FAILURE);
3938 : /* LCOV_EXCL_STOP */
3939 : }
3940 :
3941 1012 : i = i->next;
3942 : }
3943 151 : }
3944 :
3945 151 : void state_write(struct snapraid_state* state)
3946 : {
3947 : uint32_t crc;
3948 :
3949 : /* write all the content files */
3950 151 : state_write_content(state, &crc);
3951 :
3952 : /* verify the just written files */
3953 151 : state_verify_content(state, crc);
3954 :
3955 : /* rename the new files, over the old ones */
3956 151 : state_rename_content(state);
3957 :
3958 151 : state->need_write = 0; /* no write needed anymore */
3959 151 : state->checked_read = 0; /* what we wrote is not checked in read */
3960 151 : }
3961 :
3962 122 : void state_skip(struct snapraid_state* state)
3963 : {
3964 : tommy_node* i;
3965 :
3966 : /* for each disk */
3967 850 : for (i = state->disklist; i != 0; i = i->next) {
3968 : tommy_node* j;
3969 728 : struct snapraid_disk* disk = i->data;
3970 :
3971 728 : if (!disk->skip_access)
3972 727 : continue;
3973 :
3974 : /* for each file */
3975 1885 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
3976 1884 : struct snapraid_file* file = j->data;
3977 1884 : file_flag_set(file, FILE_IS_EXCLUDED);
3978 : }
3979 :
3980 : /* for each link */
3981 57 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
3982 56 : struct snapraid_link* slink = j->data;
3983 56 : link_flag_set(slink, FILE_IS_EXCLUDED);
3984 : }
3985 :
3986 : /* for each dir */
3987 1 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
3988 0 : struct snapraid_dir* dir = j->data;
3989 0 : dir_flag_set(dir, FILE_IS_EXCLUDED);
3990 : }
3991 : }
3992 122 : }
3993 :
3994 122 : void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
3995 : {
3996 : tommy_node* i;
3997 : unsigned l;
3998 :
3999 : /* if no filter, include all */
4000 122 : if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
4001 113 : return;
4002 :
4003 9 : msg_progress("Filtering...\n");
4004 :
4005 17 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4006 8 : struct snapraid_filter* filter = i->data;
4007 8 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4008 : }
4009 12 : for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4010 3 : struct snapraid_filter* filter = i->data;
4011 3 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4012 : }
4013 9 : if (filter_missing)
4014 2 : msg_verbose("\t<missing>\n");
4015 9 : if (filter_error)
4016 2 : msg_verbose("\t<error>\n");
4017 :
4018 : /* for each disk */
4019 63 : for (i = state->disklist; i != 0; i = i->next) {
4020 : tommy_node* j;
4021 54 : struct snapraid_disk* disk = i->data;
4022 :
4023 : /* if we filter for presence, we have to access the disk, so better to print something */
4024 54 : if (filter_missing)
4025 12 : msg_progress("Scanning disk %s...\n", disk->name);
4026 :
4027 : /* for each file */
4028 133944 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4029 133890 : struct snapraid_file* file = j->data;
4030 :
4031 133890 : if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4032 80982 : || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4033 78138 : || filter_existence(filter_missing, disk->dir, file->sub) != 0
4034 38646 : || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4035 : ) {
4036 109509 : file_flag_set(file, FILE_IS_EXCLUDED);
4037 : }
4038 : }
4039 :
4040 : /* for each link */
4041 4414 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4042 4360 : struct snapraid_link* slink = j->data;
4043 :
4044 4360 : if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4045 2641 : || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4046 2536 : || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4047 : ) {
4048 3221 : link_flag_set(slink, FILE_IS_EXCLUDED);
4049 : }
4050 : }
4051 :
4052 : /* for each empty dir */
4053 82 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4054 28 : struct snapraid_dir* dir = j->data;
4055 :
4056 28 : if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4057 17 : || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4058 17 : || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4059 : ) {
4060 19 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4061 : }
4062 : }
4063 : }
4064 :
4065 : /* if we are filtering by disk, exclude any parity not explicitely included */
4066 9 : if (!tommy_list_empty(filterlist_disk)) {
4067 : /* for each parity disk */
4068 20 : for (l = 0; l < state->level; ++l) {
4069 : /* check if the parity is excluded by name */
4070 16 : if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4071 : /* excluded the parity from further operation */
4072 11 : state->parity[l].is_excluded_by_filter = 1;
4073 : }
4074 : }
4075 : } else {
4076 : /* if we are filtering by file, exclude all parity */
4077 5 : if (filter_missing || !tommy_list_empty(filterlist_file)) {
4078 : /* for each parity disk */
4079 16 : for (l = 0; l < state->level; ++l) {
4080 13 : state->parity[l].is_excluded_by_filter = 1;
4081 : }
4082 : }
4083 : }
4084 : }
4085 :
4086 221 : int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4087 : {
4088 : time_t now;
4089 :
4090 221 : if (state->opt.gui) {
4091 5 : log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4092 5 : log_flush();
4093 : }
4094 :
4095 221 : now = time(0);
4096 :
4097 221 : state->progress_whole_start = now;
4098 :
4099 221 : state->progress_tick = 0;
4100 221 : state->progress_ptr = 0;
4101 221 : state->progress_wasted = 0;
4102 :
4103 : /* stop if requested */
4104 221 : if (global_interrupt) {
4105 : /* LCOV_EXCL_START */
4106 : if (!state->opt.gui) {
4107 : msg_status("Not starting for interruption\n");
4108 : }
4109 : log_tag("sigint:0: SIGINT received\n");
4110 : log_flush();
4111 : return 0;
4112 : /* LCOV_EXCL_STOP */
4113 : }
4114 :
4115 221 : return 1;
4116 : }
4117 :
4118 221 : void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4119 : {
4120 221 : if (state->opt.gui) {
4121 5 : log_tag("run:end\n");
4122 5 : log_flush();
4123 216 : } else if (countmax == 0) {
4124 19 : msg_status("Nothing to do\n");
4125 : } else {
4126 : time_t now;
4127 : time_t elapsed;
4128 :
4129 197 : unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4130 :
4131 197 : now = time(0);
4132 :
4133 197 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4134 :
4135 197 : msg_bar("%u%% completed, %u MB accessed", countpos * 100 / countmax, countsize_MB);
4136 :
4137 197 : msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4138 :
4139 : /* some extra spaces to ensure to overwrite the latest status line */
4140 197 : msg_bar(" ");
4141 :
4142 197 : msg_bar("\n");
4143 197 : msg_flush();
4144 : }
4145 221 : }
4146 :
4147 1 : void state_progress_stop(struct snapraid_state* state)
4148 : {
4149 : time_t now;
4150 :
4151 1 : now = time(0);
4152 :
4153 1 : if (!state->opt.gui) {
4154 1 : msg_bar("\n");
4155 1 : msg_flush();
4156 : }
4157 :
4158 1 : state->progress_interruption = now;
4159 1 : }
4160 :
4161 1 : void state_progress_restart(struct snapraid_state* state)
4162 : {
4163 : time_t now;
4164 :
4165 1 : now = time(0);
4166 :
4167 : /* reset the progress counter */
4168 1 : state->progress_tick = 0;
4169 1 : state->progress_ptr = 0;
4170 :
4171 1 : if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4172 1 : state->progress_wasted += now - state->progress_interruption;
4173 1 : }
4174 :
4175 : #define PROGRESS_CLEAR " "
4176 :
4177 : /**
4178 : * Set the latest tick data in the progress vector.
4179 : */
4180 353 : static void state_progress_latest(struct snapraid_state* state)
4181 : {
4182 : tommy_node* i;
4183 : unsigned l;
4184 :
4185 353 : state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4186 353 : state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4187 353 : state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4188 353 : state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4189 353 : state->progress_tick_io[state->progress_ptr] = state->tick_io;
4190 2465 : for (i = state->disklist; i != 0; i = i->next) {
4191 2112 : struct snapraid_disk* disk = i->data;
4192 2112 : disk->progress_tick[state->progress_ptr] = disk->tick;
4193 : }
4194 2021 : for (l = 0; l < state->level; ++l)
4195 1668 : state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4196 353 : }
4197 :
4198 : /**
4199 : * Number of columns
4200 : */
4201 : #define GRAPH_COLUMN 68
4202 :
4203 : /**
4204 : * Get the reference value, 0 if index is PROGRESS_MAX
4205 : */
4206 : #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4207 :
4208 2 : static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4209 : {
4210 : uint64_t v;
4211 : uint64_t tick_total;
4212 : unsigned bar;
4213 : tommy_node* i;
4214 : unsigned l;
4215 : size_t pad;
4216 :
4217 2 : tick_total = 0;
4218 :
4219 2 : tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4220 2 : tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4221 2 : tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4222 2 : tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4223 2 : tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4224 2 : if (!tick_total)
4225 0 : return;
4226 :
4227 2 : pad = 4;
4228 14 : for (i = state->disklist; i != 0; i = i->next) {
4229 12 : struct snapraid_disk* disk = i->data;
4230 12 : size_t len = strlen(disk->name);
4231 12 : if (pad < len)
4232 2 : pad = len;
4233 : }
4234 9 : for (l = 0; l < state->level; ++l) {
4235 7 : size_t len = strlen(lev_config_name(l));
4236 7 : if (pad < len)
4237 3 : pad = len;
4238 : }
4239 :
4240 : /* extra space */
4241 2 : pad += 1;
4242 :
4243 2 : if (pad + 30 < GRAPH_COLUMN)
4244 2 : bar = GRAPH_COLUMN - pad;
4245 : else
4246 0 : bar = 30;
4247 :
4248 2 : if (io) {
4249 1 : const char* legend = "cached blocks (instant, more is better)";
4250 :
4251 : /* refresh the cached blocks info */
4252 1 : io_refresh(io);
4253 :
4254 1 : printf("\n");
4255 :
4256 7 : for (i = state->disklist; i != 0; i = i->next) {
4257 6 : struct snapraid_disk* disk = i->data;
4258 6 : v = disk->cached;
4259 6 : printr(disk->name, pad);
4260 6 : printf("%4" PRIu64 " | ", v);
4261 6 : printc('o', v * bar / io->io_max);
4262 6 : printf("\n");
4263 : }
4264 :
4265 2 : for (l = 0; l < state->level; ++l) {
4266 1 : v = state->parity[l].cached;
4267 1 : printr(lev_config_name(l), pad);
4268 1 : printf("%4" PRIu64 " | ", v);
4269 1 : printc('o', v * bar / io->io_max);
4270 1 : printf("\n");
4271 : }
4272 :
4273 1 : printc(' ', pad);
4274 1 : printf(" |_");
4275 1 : printc('_', bar);
4276 1 : printf("\n");
4277 :
4278 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4279 1 : printf("%s", legend);
4280 1 : printf("\n");
4281 : }
4282 :
4283 2 : printf("\n");
4284 :
4285 14 : for (i = state->disklist; i != 0; i = i->next) {
4286 12 : struct snapraid_disk* disk = i->data;
4287 12 : v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4288 12 : printr(disk->name, pad);
4289 12 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4290 12 : printc('*', v * bar / tick_total);
4291 12 : printf("\n");
4292 : }
4293 :
4294 9 : for (l = 0; l < state->level; ++l) {
4295 7 : v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4296 7 : printr(lev_config_name(l), pad);
4297 7 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4298 7 : printc('*', v * bar / tick_total);
4299 7 : printf("\n");
4300 : }
4301 :
4302 2 : v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4303 2 : printr("raid", pad);
4304 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4305 2 : printc('*', v * bar / tick_total);
4306 2 : printf("\n");
4307 :
4308 2 : v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4309 2 : printr("hash", pad);
4310 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4311 2 : printc('*', v * bar / tick_total);
4312 2 : printf("\n");
4313 :
4314 2 : v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4315 2 : printr("sched", pad);
4316 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4317 2 : printc('*', v * bar / tick_total);
4318 2 : printf("\n");
4319 :
4320 2 : v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4321 2 : printr("misc", pad);
4322 2 : printf("%3" PRIu64 "%% | ", v * 100 / tick_total);
4323 2 : printc('*', v * bar / tick_total);
4324 2 : printf("\n");
4325 :
4326 2 : printc(' ', pad);
4327 2 : printf(" |_");
4328 2 : printc('_', bar);
4329 2 : printf("\n");
4330 :
4331 2 : if (oldest == PROGRESS_MAX) {
4332 1 : const char* legend = "wait time (total, less is better)";
4333 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4334 1 : printf("%s", legend);
4335 1 : printf("\n");
4336 : } else {
4337 1 : const char* legend_d = "wait time (last %d secs, less is better)";
4338 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
4339 1 : printf(legend_d, PROGRESS_MAX);
4340 1 : printf("\n");
4341 : }
4342 :
4343 2 : printf("\n");
4344 : }
4345 :
4346 1073464 : int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4347 : {
4348 : time_t now;
4349 : int pred;
4350 :
4351 1073464 : now = time(0);
4352 :
4353 : /* previous position */
4354 1073464 : pred = state->progress_ptr + PROGRESS_MAX - 1;
4355 1073464 : if (pred >= PROGRESS_MAX)
4356 1073265 : pred -= PROGRESS_MAX;
4357 :
4358 : /* if the previous measure is different */
4359 1073464 : if (state->progress_tick == 0
4360 1073265 : || state->progress_time[pred] != now
4361 : ) {
4362 : time_t elapsed;
4363 257 : unsigned out_perc = 0;
4364 257 : unsigned out_speed = 0;
4365 257 : unsigned out_cpu = 0;
4366 257 : unsigned out_eta = 0;
4367 257 : int out_computed = 0;
4368 :
4369 : /* store the new measure */
4370 257 : state->progress_time[state->progress_ptr] = now;
4371 257 : state->progress_pos[state->progress_ptr] = countpos;
4372 257 : state->progress_size[state->progress_ptr] = countsize;
4373 257 : state_progress_latest(state);
4374 :
4375 257 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4376 :
4377 : /* completion percentage */
4378 257 : if (countmax)
4379 257 : out_perc = countpos * 100 / countmax;
4380 :
4381 : /* if we have at least 5 measures */
4382 257 : if (state->progress_tick >= 5
4383 : /* or if we are running in test mode, with at least one measure */
4384 257 : || (state->opt.force_progress && state->progress_tick >= 1)
4385 : ) {
4386 : int oldest;
4387 : int past;
4388 : time_t delta_time;
4389 : block_off_t delta_pos;
4390 : data_off_t delta_size;
4391 : uint64_t tick_cpu;
4392 : uint64_t tick_total;
4393 : uint64_t delta_tick_cpu;
4394 : uint64_t delta_tick_total;
4395 : uint64_t oldest_tick_cpu;
4396 : uint64_t oldest_tick_total;
4397 :
4398 : /* number of past measures */
4399 58 : past = state->progress_tick;
4400 :
4401 : /* drop the oldest ones, to promptly */
4402 : /* skip the startup phase */
4403 58 : past -= past / 5;
4404 :
4405 : /* check how much we can go in the past */
4406 58 : if (past >= PROGRESS_MAX - 1) {
4407 : /* the vector is filled, so we are already in position */
4408 : /* to get the possible oldest one */
4409 0 : oldest = state->progress_ptr + 1;
4410 : } else {
4411 : /* go backward the number of positions selected */
4412 58 : oldest = state->progress_ptr + PROGRESS_MAX - past;
4413 : }
4414 58 : if (oldest >= PROGRESS_MAX)
4415 58 : oldest -= PROGRESS_MAX;
4416 :
4417 116 : tick_cpu = state->progress_tick_misc[state->progress_ptr]
4418 58 : + state->progress_tick_sched[state->progress_ptr]
4419 58 : + state->progress_tick_raid[state->progress_ptr]
4420 58 : + state->progress_tick_hash[state->progress_ptr];
4421 58 : tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
4422 :
4423 116 : oldest_tick_cpu = state->progress_tick_misc[oldest]
4424 58 : + state->progress_tick_sched[oldest]
4425 58 : + state->progress_tick_raid[oldest]
4426 58 : + state->progress_tick_hash[oldest];
4427 58 : oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
4428 :
4429 58 : delta_time = now - state->progress_time[oldest];
4430 58 : delta_pos = countpos - state->progress_pos[oldest];
4431 58 : delta_size = countsize - state->progress_size[oldest];
4432 58 : delta_tick_cpu = tick_cpu - oldest_tick_cpu;
4433 58 : delta_tick_total = tick_total - oldest_tick_total;
4434 :
4435 : /* estimate the speed in MB/s */
4436 58 : if (delta_time != 0)
4437 58 : out_speed = (unsigned)(delta_size / MEGA / delta_time);
4438 :
4439 : /* estimate the cpu usage percentage */
4440 58 : if (delta_tick_total != 0)
4441 10 : out_cpu = (unsigned)(delta_tick_cpu * 100U / delta_tick_total);
4442 :
4443 : /* estimate the remaining time in minutes */
4444 58 : if (delta_pos != 0)
4445 58 : out_eta = (countmax - countpos) * delta_time / (60 * delta_pos);
4446 :
4447 58 : if (state->opt.force_stats) {
4448 1 : os_clear();
4449 1 : state_progress_graph(state, io, state->progress_ptr, oldest);
4450 : }
4451 :
4452 : /* we have the output value */
4453 58 : out_computed = 1;
4454 : }
4455 :
4456 257 : if (state->opt.gui) {
4457 6 : log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed);
4458 6 : log_flush();
4459 : } else {
4460 251 : msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
4461 251 : if (out_computed) {
4462 57 : msg_bar(", %u MB/s", out_speed);
4463 57 : msg_bar(", CPU %u%%", out_cpu);
4464 57 : msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
4465 : }
4466 251 : msg_bar("%s\r", PROGRESS_CLEAR);
4467 251 : msg_flush();
4468 : }
4469 :
4470 : /* next position to fill */
4471 257 : ++state->progress_ptr;
4472 257 : if (state->progress_ptr >= PROGRESS_MAX)
4473 0 : state->progress_ptr -= PROGRESS_MAX;
4474 :
4475 : /* one more measure */
4476 257 : ++state->progress_tick;
4477 : }
4478 :
4479 : /* stop if requested */
4480 1073464 : if (global_interrupt) {
4481 : /* LCOV_EXCL_START */
4482 : if (!state->opt.gui) {
4483 : log_fatal("\n");
4484 : log_fatal("Stopping for interruption at block %u\n", blockpos);
4485 : }
4486 : log_tag("sigint:%u: SIGINT received\n", blockpos);
4487 : log_flush();
4488 : return 1;
4489 : /* LCOV_EXCL_STOP */
4490 : }
4491 :
4492 1073464 : return 0;
4493 : }
4494 :
4495 104 : void state_usage_waste(struct snapraid_state* state)
4496 : {
4497 104 : uint64_t now = tick();
4498 :
4499 104 : state->tick_last = now;
4500 104 : }
4501 :
4502 1063434 : void state_usage_misc(struct snapraid_state* state)
4503 : {
4504 1063434 : uint64_t now = tick();
4505 1063434 : uint64_t delta = now - state->tick_last;
4506 :
4507 : /* increment the time spent in computations */
4508 1063434 : state->tick_misc += delta;
4509 :
4510 1063434 : state->tick_last = now;
4511 1063434 : }
4512 :
4513 149890 : void state_usage_sched(struct snapraid_state* state)
4514 : {
4515 149890 : uint64_t now = tick();
4516 149890 : uint64_t delta = now - state->tick_last;
4517 :
4518 : /* increment the time spent in computations */
4519 149890 : state->tick_sched += delta;
4520 :
4521 149890 : state->tick_last = now;
4522 149890 : }
4523 :
4524 140325 : void state_usage_raid(struct snapraid_state* state)
4525 : {
4526 140325 : uint64_t now = tick();
4527 140325 : uint64_t delta = now - state->tick_last;
4528 :
4529 : /* increment the time spent in computations */
4530 140325 : state->tick_raid += delta;
4531 :
4532 140325 : state->tick_last = now;
4533 140325 : }
4534 :
4535 777134 : void state_usage_hash(struct snapraid_state* state)
4536 : {
4537 777134 : uint64_t now = tick();
4538 777134 : uint64_t delta = now - state->tick_last;
4539 :
4540 : /* increment the time spent in computations */
4541 777134 : state->tick_hash += delta;
4542 :
4543 777134 : state->tick_last = now;
4544 777134 : }
4545 :
4546 912681 : void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
4547 : {
4548 912681 : uint64_t now = tick();
4549 912681 : uint64_t delta = now - state->tick_last;
4550 : unsigned i;
4551 :
4552 : /* increment the time spent in the data disks */
4553 956391 : for (i = 0; i < waiting_mac; ++i) {
4554 43710 : struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
4555 :
4556 43710 : if (!disk)
4557 2 : continue;
4558 :
4559 43708 : disk->tick += delta;
4560 : }
4561 912681 : state->tick_io += delta;
4562 :
4563 912681 : state->tick_last = now;
4564 912681 : }
4565 :
4566 774353 : void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
4567 : {
4568 774353 : uint64_t now = tick();
4569 774353 : uint64_t delta = now - state->tick_last;
4570 : unsigned i;
4571 :
4572 : /* increment the time spent in the parity disk */
4573 784293 : for (i = 0; i < waiting_mac; ++i)
4574 9940 : state->parity[waiting_map[i]].tick += delta;
4575 774353 : state->tick_io += delta;
4576 :
4577 774353 : state->tick_last = now;
4578 774353 : }
4579 :
4580 96 : void state_usage_print(struct snapraid_state* state)
4581 : {
4582 : /* set the latest data */
4583 96 : state_progress_latest(state);
4584 :
4585 96 : if (msg_level < MSG_PROGRESS)
4586 95 : return;
4587 :
4588 : /* print a graph for it */
4589 1 : state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
4590 : }
4591 :
4592 499 : void state_fscheck(struct snapraid_state* state, const char* ope)
4593 : {
4594 : tommy_node* i;
4595 :
4596 : /* check the file-system on all disks */
4597 3487 : for (i = state->disklist; i != 0; i = i->next) {
4598 2988 : struct snapraid_disk* disk = i->data;
4599 :
4600 2988 : if (fs_check(disk) != 0) {
4601 : /* LCOV_EXCL_START */
4602 : log_fatal("Internal inconsistency in file-system for disk '%s' %s\n", disk->name, ope);
4603 : os_abort();
4604 : /* LCOV_EXCL_STOP */
4605 : }
4606 : }
4607 499 : }
4608 :
4609 1 : void generate_configuration(const char* path)
4610 : {
4611 : struct snapraid_state state;
4612 : struct snapraid_content* content;
4613 : char esc_buffer[ESC_MAX];
4614 : unsigned l, s;
4615 : tommy_node* j;
4616 :
4617 1 : state_init(&state);
4618 :
4619 : /* mark that we are without a configuration file */
4620 1 : state.no_conf = 1;
4621 :
4622 : /* create the dummy content entry */
4623 1 : content = content_alloc(path, -1);
4624 :
4625 : /* adds the content entry */
4626 1 : tommy_list_insert_tail(&state.contentlist, &content->node, content);
4627 :
4628 : /* read the content file */
4629 1 : state_read(&state);
4630 :
4631 : /* output a dummy configuration file */
4632 1 : printf("# Configuration file generated from %s\n", path);
4633 1 : printf("\n");
4634 1 : printf("# Use this blocksize\n");
4635 1 : printf("blocksize %u\n", state.block_size / KIBI);
4636 1 : printf("\n");
4637 1 : printf("# Use this hashsize\n");
4638 1 : printf("hashsize %u\n", BLOCK_HASH_SIZE);
4639 1 : printf("\n");
4640 7 : for (l = 0; l < state.level; ++l) {
4641 6 : printf("# Set the correct path for the %s files\n", lev_name(l));
4642 6 : printf("# You had %u of them:\n", state.parity[l].split_mac);
4643 30 : for (s = 0; s < state.parity[l].split_mac; ++s) {
4644 24 : printf("# %u:\n", s);
4645 24 : printf("# PATH:");
4646 24 : if (state.parity[l].split_map[s].path[0])
4647 24 : printf("%s", state.parity[l].split_map[s].path);
4648 : else
4649 0 : printf("?");
4650 24 : printf("\n");
4651 24 : printf("# SIZE:");
4652 24 : if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
4653 24 : printf("%" PRIu64, state.parity[l].split_map[s].size);
4654 : else
4655 0 : printf("?");
4656 24 : printf("\n");
4657 24 : printf("# UUID:");
4658 24 : if (state.parity[l].split_map[s].uuid[0])
4659 24 : printf("%s", state.parity[l].split_map[s].uuid);
4660 : else
4661 0 : printf("?");
4662 24 : printf("\n");
4663 24 : printf("#\n");
4664 : }
4665 6 : printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
4666 6 : printf("\n");
4667 : }
4668 1 : printf("# Add any other content file\n");
4669 1 : printf("content %s\n", path);
4670 1 : printf("\n");
4671 7 : for (j = state.maplist; j; j = j->next) {
4672 6 : struct snapraid_map* map = j->data;
4673 : struct snapraid_disk* disk;
4674 6 : printf("# Set the correct dir for disk '%s'\n", map->name);
4675 6 : if (map->uuid[0])
4676 6 : printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
4677 6 : disk = find_disk_by_name(&state, map->name);
4678 6 : if (disk && disk->filelist) {
4679 6 : struct snapraid_file* file = disk->filelist->data;
4680 6 : if (file) {
4681 6 : printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
4682 : }
4683 : }
4684 6 : printf("data %s ENTER_HERE_THE_DIR\n", map->name);
4685 6 : printf("\n");
4686 : }
4687 :
4688 1 : state_done(&state);
4689 1 : }
4690 :
|
