Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "elem.h"
21 : #include "import.h"
22 : #include "search.h"
23 : #include "state.h"
24 : #include "support.h"
25 : #include "parity.h"
26 : #include "stream.h"
27 : #include "handle.h"
28 : #include "io.h"
29 : #include "thermal.h"
30 : #include "raid/raid.h"
31 : #include "raid/cpu.h"
32 :
33 : /**
34 : * Configure the multithread support.
35 : *
36 : * Multi thread for write could be either faster or slower, depending
37 : * on the specific conditions. With multithreads it's likely faster
38 : * writing to disk, but you'll need to access multiple times the same data,
39 : * being potentially slower.
40 : *
41 : * Multi thread for verify is instead always generally faster,
42 : * so we enable it if possible.
43 : */
44 : #if HAVE_THREAD
45 : /* #define HAVE_MT_WRITE 1 */
46 : #define HAVE_MT_VERIFY 1
47 : #endif
48 :
49 7 : const char* lev_name(unsigned l)
50 : {
51 7 : switch (l) {
52 2 : case 0 : return "Parity";
53 1 : case 1 : return "2-Parity";
54 1 : case 2 : return "3-Parity";
55 1 : case 3 : return "4-Parity";
56 1 : case 4 : return "5-Parity";
57 1 : case 5 : return "6-Parity";
58 : }
59 :
60 0 : return 0;
61 : }
62 :
63 261647 : const char* lev_config_name(unsigned l)
64 : {
65 261647 : switch (l) {
66 91955 : case 0 : return "parity";
67 74618 : case 1 : return "2-parity";
68 23808 : case 2 : return "3-parity";
69 23783 : case 3 : return "4-parity";
70 23758 : case 4 : return "5-parity";
71 23725 : case 5 : return "6-parity";
72 : }
73 :
74 0 : return 0;
75 : }
76 :
77 6721 : static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
78 : {
79 6721 : if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
80 635 : *level = 0;
81 635 : return 0;
82 : }
83 :
84 6086 : if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
85 227 : *level = 1;
86 227 : return 0;
87 : }
88 :
89 5859 : if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
90 202 : *level = 2;
91 202 : return 0;
92 : }
93 :
94 5657 : if (strcmp(s, "4-parity") == 0) {
95 199 : *level = 3;
96 199 : return 0;
97 : }
98 :
99 5458 : if (strcmp(s, "5-parity") == 0) {
100 196 : *level = 4;
101 196 : return 0;
102 : }
103 :
104 5262 : if (strcmp(s, "6-parity") == 0) {
105 192 : *level = 5;
106 192 : return 0;
107 : }
108 :
109 5070 : if (strcmp(s, "z-parity") == 0) {
110 0 : *level = 2;
111 0 : if (mode)
112 0 : *mode = RAID_MODE_VANDERMONDE;
113 0 : return 0;
114 : }
115 :
116 5070 : return -1;
117 : }
118 :
119 267 : const char* lev_raid_name(unsigned mode, unsigned n)
120 : {
121 267 : switch (n) {
122 41 : case 1 : return "par1";
123 25 : case 2 : return "par2";
124 3 : case 3 : if (mode == RAID_MODE_CAUCHY)
125 3 : return "par3";
126 : else
127 0 : return "parz";
128 3 : case 4 : return "par4";
129 4 : case 5 : return "par5";
130 191 : case 6 : return "par6";
131 : }
132 :
133 0 : return 0;
134 : }
135 :
136 269 : void state_init(struct snapraid_state* state)
137 : {
138 : unsigned l, s, i;
139 :
140 269 : memset(&state->opt, 0, sizeof(state->opt));
141 269 : state->filter_hidden = 0;
142 269 : state->autosave = 0;
143 269 : state->need_write = 0;
144 269 : state->written = 0;
145 269 : state->checked_read = 0;
146 269 : state->block_size = 256 * KIBI; /* default 256 KiB */
147 269 : state->raid_mode = RAID_MODE_CAUCHY;
148 269 : state->file_mode = ADVISE_DEFAULT;
149 1883 : for (l = 0; l < LEV_MAX; ++l) {
150 1614 : state->parity[l].split_mac = 0;
151 14526 : for (s = 0; s < SPLIT_MAX; ++s) {
152 12912 : state->parity[l].split_map[s].path[0] = 0;
153 12912 : state->parity[l].split_map[s].uuid[0] = 0;
154 12912 : state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
155 12912 : state->parity[l].split_map[s].device = 0;
156 : }
157 1614 : state->parity[l].smartctl[0] = 0;
158 8070 : for (i = 0; i < SMART_IGNORE_MAX; ++i)
159 6456 : state->parity[l].smartignore[i] = 0;
160 1614 : state->parity[l].total_blocks = 0;
161 1614 : state->parity[l].free_blocks = 0;
162 1614 : state->parity[l].skip_access = 0;
163 1614 : state->parity[l].tick = 0;
164 1614 : state->parity[l].cached_blocks = 0;
165 1614 : state->parity[l].is_excluded_by_filter = 0;
166 : }
167 269 : state->tick_io = 0;
168 269 : state->tick_misc = 0;
169 269 : state->tick_sched = 0;
170 269 : state->tick_raid = 0;
171 269 : state->tick_hash = 0;
172 269 : state->tick_last = tick();
173 269 : state->share[0] = 0;
174 269 : state->pool[0] = 0;
175 269 : state->pool_device = 0;
176 269 : state->lockfile[0] = 0;
177 269 : state->level = 1; /* default is the lowest protection */
178 269 : state->clear_past_hash = 0;
179 269 : state->no_conf = 0;
180 1345 : for (i = 0; i < SMART_IGNORE_MAX; ++i)
181 1076 : state->smartignore[i] = 0;
182 269 : state->thermal_stop_gathering = 0;
183 269 : state->thermal_ambient_temperature = 0;
184 269 : state->thermal_highest_temperature = 0;
185 269 : state->thermal_first = 0;
186 269 : state->thermal_latest = 0;
187 269 : state->thermal_temperature_limit = 0;
188 269 : state->thermal_cooldown_time = 0;
189 :
190 269 : tommy_list_init(&state->disklist);
191 269 : tommy_list_init(&state->maplist);
192 269 : tommy_list_init(&state->contentlist);
193 269 : tommy_list_init(&state->filterlist);
194 269 : tommy_list_init(&state->importlist);
195 269 : tommy_list_init(&state->thermallist);
196 269 : tommy_hashdyn_init(&state->importset);
197 269 : tommy_hashdyn_init(&state->previmportset);
198 269 : tommy_hashdyn_init(&state->searchset);
199 269 : tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
200 269 : }
201 :
202 258 : void state_done(struct snapraid_state* state)
203 : {
204 258 : tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
205 258 : tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
206 258 : tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
207 258 : tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
208 258 : tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
209 258 : tommy_list_foreach(&state->thermallist, (tommy_foreach_func*)thermal_free);
210 258 : tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
211 258 : tommy_hashdyn_done(&state->importset);
212 258 : tommy_hashdyn_done(&state->previmportset);
213 258 : tommy_hashdyn_done(&state->searchset);
214 258 : tommy_arrayblkof_done(&state->infoarr);
215 258 : }
216 :
217 : /**
218 : * Check the configuration.
219 : */
220 267 : static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
221 : {
222 : tommy_node* i;
223 : unsigned l, s;
224 :
225 : /* check for parity level */
226 267 : if (state->raid_mode == RAID_MODE_VANDERMONDE) {
227 0 : if (state->level > 3) {
228 : /* LCOV_EXCL_START */
229 : log_fatal("Using z-parity limits you to a maximum of 3 parities.\n");
230 : exit(EXIT_FAILURE);
231 : /* LCOV_EXCL_STIO */
232 : }
233 : }
234 :
235 : for (l = 0; l < state->level; ++l) {
236 : if (state->parity[l].split_mac == 0) {
237 : /* LCOV_EXCL_START */
238 : log_fatal("Missing '%s' specification in '%s'\n", lev_config_name(l), path);
239 : exit(EXIT_FAILURE);
240 : /* LCOV_EXCL_STOP */
241 : }
242 : }
243 :
244 267 : if (tommy_list_empty(&state->contentlist)) {
245 : /* LCOV_EXCL_START */
246 : log_fatal("Missing 'content' specification in '%s'\n", path);
247 : exit(EXIT_FAILURE);
248 : /* LCOV_EXCL_STOP */
249 : }
250 :
251 : /* check for equal paths */
252 1812 : for (i = state->contentlist; i != 0; i = i->next) {
253 1545 : struct snapraid_content* content = i->data;
254 :
255 10015 : for (l = 0; l < state->level; ++l) {
256 42350 : for (s = 0; s < state->parity[l].split_mac; ++s) {
257 33880 : if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
258 : /* LCOV_EXCL_START */
259 : log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
260 : exit(EXIT_FAILURE);
261 : /* LCOV_EXCL_STOP */
262 : }
263 : }
264 : }
265 : }
266 :
267 : /* check device of data disks */
268 267 : if (!state->opt.skip_device && !state->opt.skip_disk_access) {
269 0 : for (i = state->disklist; i != 0; i = i->next) {
270 : tommy_node* j;
271 0 : struct snapraid_disk* disk = i->data;
272 :
273 : /* skip data disks that are not accessible */
274 0 : if (disk->skip_access)
275 0 : continue;
276 :
277 : #ifdef _WIN32
278 : if (disk->device == 0) {
279 : /* LCOV_EXCL_START */
280 : log_fatal("Disk '%s' has a zero serial number.\n", disk->dir);
281 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
282 : log_fatal("it's better to change the serial number of the disk.\n");
283 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
284 : exit(EXIT_FAILURE);
285 : /* LCOV_EXCL_STOP */
286 : }
287 : #endif
288 :
289 0 : for (j = i->next; j != 0; j = j->next) {
290 0 : struct snapraid_disk* other = j->data;
291 0 : if (disk->device == other->device) {
292 0 : if (state->opt.force_device) {
293 : /* note that we just ignore the issue */
294 : /* and we DON'T mark the disk to be skipped */
295 : /* because we want to use these disks */
296 0 : if (!state->opt.no_warnings)
297 0 : log_fatal("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
298 : } else {
299 : /* LCOV_EXCL_START */
300 : log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
301 : #ifdef _WIN32
302 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
303 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
304 : log_fatal("to change one of the disk serial.\n");
305 : #endif
306 : /* in "fix" we allow to continue anyway */
307 : if (strcmp(state->command, "fix") == 0) {
308 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
309 : }
310 : exit(EXIT_FAILURE);
311 : /* LCOV_EXCL_STOP */
312 : }
313 : }
314 : }
315 :
316 : /* skip data disks that are not accessible */
317 0 : if (disk->skip_access)
318 0 : continue;
319 :
320 0 : if (!state->opt.skip_parity_access) {
321 0 : for (l = 0; l < state->level; ++l) {
322 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
323 0 : if (disk->device == state->parity[l].split_map[s].device) {
324 0 : if (state->opt.force_device) {
325 : /* note that we just ignore the issue */
326 : /* and we DON'T mark the disk to be skipped */
327 : /* because we want to use these disks */
328 0 : if (!state->opt.no_warnings)
329 0 : log_fatal("DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
330 : } else {
331 : /* LCOV_EXCL_START */
332 : log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
333 : #ifdef _WIN32
334 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
335 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
336 : log_fatal("to change one of the disk serial.\n");
337 : #endif
338 : exit(EXIT_FAILURE);
339 : /* LCOV_EXCL_STOP */
340 : }
341 : }
342 : }
343 : }
344 : }
345 :
346 0 : if (state->pool[0] != 0 && disk->device == state->pool_device) {
347 : /* LCOV_EXCL_START */
348 : log_fatal("Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
349 : #ifdef _WIN32
350 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
351 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
352 : log_fatal("to change one of the disk serial.\n");
353 : #endif
354 : exit(EXIT_FAILURE);
355 : /* LCOV_EXCL_STOP */
356 : }
357 : }
358 : }
359 :
360 : /* check device of parity disks */
361 267 : if (!state->opt.skip_device && !state->opt.skip_parity_access) {
362 0 : for (l = 0; l < state->level; ++l) {
363 0 : for (s = 0; s < state->parity[l].split_mac; ++s) {
364 : unsigned j, t;
365 :
366 : /* skip parity disks that are not accessible */
367 0 : if (state->parity[l].skip_access)
368 0 : continue;
369 :
370 : #ifdef _WIN32
371 : if (state->parity[l].split_map[s].device == 0) {
372 : /* LCOV_EXCL_START */
373 : log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
374 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
375 : log_fatal("it's better to change the serial number of the disk.\n");
376 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
377 : exit(EXIT_FAILURE);
378 : /* LCOV_EXCL_STOP */
379 : }
380 : #endif
381 :
382 0 : for (j = l + 1; j < state->level; ++j) {
383 0 : for (t = 0; t < state->parity[j].split_mac; ++t) {
384 0 : if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
385 0 : if (state->opt.force_device) {
386 : /* note that we just ignore the issue */
387 : /* and we DON'T mark the disk to be skipped */
388 : /* because we want to use these disks */
389 0 : if (!state->opt.no_warnings)
390 0 : log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
391 : } else {
392 : /* LCOV_EXCL_START */
393 : log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
394 : #ifdef _WIN32
395 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
396 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
397 : log_fatal("to change one of the disk serial.\n");
398 : #endif
399 : /* in "fix" we allow to continue anyway */
400 : if (strcmp(state->command, "fix") == 0) {
401 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
402 : }
403 : exit(EXIT_FAILURE);
404 : /* LCOV_EXCL_STOP */
405 : }
406 : }
407 : }
408 : }
409 :
410 0 : if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
411 : /* LCOV_EXCL_START */
412 : log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
413 : #ifdef _WIN32
414 : log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
415 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
416 : log_fatal("to change one of the disk serial.\n");
417 : #endif
418 : exit(EXIT_FAILURE);
419 : /* LCOV_EXCL_STOP */
420 : }
421 : }
422 : }
423 : }
424 :
425 : /* check device of pool disk */
426 : #ifdef _WIN32
427 : if (!state->opt.skip_device) {
428 : if (state->pool[0] != 0 && state->pool_device == 0) {
429 : /* LCOV_EXCL_START */
430 : log_fatal("Disk '%s' has a zero serial number.\n", state->pool);
431 : log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
432 : log_fatal("it's better to change the serial number of the disk.\n");
433 : log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
434 : exit(EXIT_FAILURE);
435 : /* LCOV_EXCL_STOP */
436 : }
437 : }
438 : #endif
439 :
440 : /* count the content files */
441 267 : if (!state->opt.skip_device && !state->opt.skip_content_access) {
442 : unsigned content_count;
443 :
444 0 : content_count = 0;
445 0 : for (i = state->contentlist; i != 0; i = i->next) {
446 : tommy_node* j;
447 0 : struct snapraid_content* content = i->data;
448 :
449 : /* check if there are others in the same disk */
450 0 : for (j = i->next; j != 0; j = j->next) {
451 0 : struct snapraid_content* other = j->data;
452 0 : if (content->device == other->device) {
453 0 : log_fatal("WARNING! Content files on the same disk: '%s' and '%s'.\n", content->content, other->content);
454 0 : break;
455 : }
456 : }
457 0 : if (j != 0) {
458 : /* skip it */
459 0 : continue;
460 : }
461 :
462 0 : ++content_count;
463 : }
464 :
465 0 : if (content_count < state->level + 1) {
466 : /* LCOV_EXCL_START */
467 : log_fatal("You must have at least %d 'content' files in different disks.\n", state->level + 1);
468 : exit(EXIT_FAILURE);
469 : /* LCOV_EXCL_STOP */
470 : }
471 : }
472 :
473 : /* check for speed */
474 : #ifdef CONFIG_X86
475 267 : if (!raid_cpu_has_ssse3())
476 : #endif
477 0 : if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
478 0 : if (state->level == 3) {
479 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
480 0 : log_fatal("WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
481 0 : } else if (state->level > 3) {
482 0 : log_fatal("WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
483 0 : log_fatal("WARNING! It's recommended to reduce the parity levels to triple parity.\n");
484 : }
485 : }
486 :
487 : /* ensure that specified filter disks are valid ones */
488 277 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
489 : tommy_node* j;
490 10 : struct snapraid_filter* filter = i->data;
491 51 : for (j = state->disklist; j != 0; j = j->next) {
492 45 : struct snapraid_disk* disk = j->data;
493 45 : if (fnmatch(filter->pattern, disk->name, FNM_CASEINSENSITIVE_FOR_WIN) == 0)
494 4 : break;
495 : }
496 10 : if (j == 0) {
497 : /* check matching with parity disks */
498 21 : for (l = 0; l < state->level; ++l)
499 21 : if (fnmatch(filter->pattern, lev_config_name(l), FNM_CASEINSENSITIVE_FOR_WIN) == 0)
500 6 : break;
501 6 : if (l == state->level) {
502 : /* LCOV_EXCL_START */
503 : log_fatal("Option -d, --filter-disk %s doesn't match any data or parity disk.\n", filter->pattern);
504 : exit(EXIT_FAILURE);
505 : /* LCOV_EXCL_STOP */
506 : }
507 : }
508 : }
509 267 : }
510 :
511 : /**
512 : * Validate the smartctl command.
513 : *
514 : * It must contains only one %s string, and not other % chars.
515 : */
516 368 : static int validate_smartctl(const char* custom)
517 : {
518 368 : const char* s = custom;
519 368 : int arg = 0;
520 :
521 2208 : while (*s) {
522 1840 : if (s[0] == '%' && s[1] == 's') {
523 184 : if (arg) {
524 : /* LCOV_EXCL_START */
525 : return -1;
526 : /* LCOV_EXCL_STOP */
527 : }
528 184 : arg = 1;
529 1656 : } else if (s[0] == '%') {
530 : /* LCOV_EXCL_START */
531 : return -1;
532 : /* LCOV_EXCL_STOP */
533 : }
534 :
535 1840 : ++s;
536 : }
537 :
538 368 : return 0;
539 : }
540 :
541 268 : void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
542 : {
543 : STREAM* f;
544 : unsigned line;
545 : tommy_node* i;
546 : unsigned l, s;
547 :
548 : /* copy the options */
549 268 : state->opt = *opt;
550 :
551 : /* if unset, sort by physical order */
552 268 : if (!state->opt.force_order)
553 1 : state->opt.force_order = SORT_PHYSICAL;
554 :
555 : /* adjust file mode */
556 268 : if (state->opt.file_mode != ADVISE_DEFAULT) {
557 267 : state->file_mode = state->opt.file_mode;
558 : } else {
559 : /* default mode, if nothing is specified */
560 1 : state->file_mode = ADVISE_DISCARD;
561 : }
562 :
563 : /* store current command */
564 268 : state->command = command;
565 :
566 268 : log_tag("conf:file:%s\n", path);
567 :
568 268 : f = sopen_read(path);
569 268 : if (!f) {
570 : /* LCOV_EXCL_START */
571 : if (errno == ENOENT) {
572 : if (state->opt.auto_conf) {
573 : log_tag("conf:missing:\n");
574 : /* mark that we are without a configuration file */
575 : state->no_conf = 1;
576 : state->level = 0;
577 : return;
578 : }
579 :
580 : log_fatal("No configuration file found at '%s'\n", path);
581 : } else if (errno == EACCES) {
582 : log_fatal("You do not have rights to access the configuration file '%s'\n", path);
583 : } else {
584 : log_fatal("Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
585 : }
586 : exit(EXIT_FAILURE);
587 : /* LCOV_EXCL_STOP */
588 : }
589 :
590 268 : line = 1;
591 6170 : while (1) {
592 : char tag[PATH_MAX];
593 : char buffer[PATH_MAX];
594 : int ret;
595 : int c;
596 : unsigned level;
597 :
598 : /* skip initial spaces */
599 6438 : sgetspace(f);
600 :
601 : /* read the command */
602 6438 : ret = sgettok(f, tag, sizeof(tag));
603 6438 : if (ret < 0) {
604 : /* LCOV_EXCL_START */
605 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
606 : exit(EXIT_FAILURE);
607 : /* LCOV_EXCL_STOP */
608 : }
609 :
610 : /* skip spaces after the command */
611 6438 : sgetspace(f);
612 :
613 6438 : if (strcmp(tag, "blocksize") == 0
614 : /* block_size is the old format of the option */
615 6170 : || strcmp(tag, "block_size") == 0) {
616 :
617 268 : ret = sgetu32(f, &state->block_size);
618 268 : if (ret < 0) {
619 : /* LCOV_EXCL_START */
620 : log_fatal("Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
621 : exit(EXIT_FAILURE);
622 : /* LCOV_EXCL_STOP */
623 : }
624 268 : if (state->block_size < 1) {
625 : /* LCOV_EXCL_START */
626 : log_fatal("Too small 'blocksize' specification in '%s' at line %u\n", path, line);
627 : exit(EXIT_FAILURE);
628 : /* LCOV_EXCL_STOP */
629 : }
630 268 : if (state->block_size > 16 * KIBI) {
631 : /* LCOV_EXCL_START */
632 : log_fatal("Too big 'blocksize' specification in '%s' at line %u\n", path, line);
633 : exit(EXIT_FAILURE);
634 : /* LCOV_EXCL_STOP */
635 : }
636 : /* check if it's a power of 2 */
637 268 : if ((state->block_size & (state->block_size - 1)) != 0) {
638 : /* LCOV_EXCL_START */
639 : log_fatal("Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
640 : exit(EXIT_FAILURE);
641 : /* LCOV_EXCL_STOP */
642 : }
643 268 : state->block_size *= KIBI;
644 6170 : } else if (strcmp(tag, "hashsize") == 0
645 6170 : || strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
646 0 : ) {
647 : uint32_t hash_size;
648 :
649 0 : ret = sgetu32(f, &hash_size);
650 0 : if (ret < 0) {
651 : /* LCOV_EXCL_START */
652 : log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
653 : exit(EXIT_FAILURE);
654 : /* LCOV_EXCL_STOP */
655 : }
656 0 : if (hash_size < 2) {
657 : /* LCOV_EXCL_START */
658 : log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
659 : exit(EXIT_FAILURE);
660 : /* LCOV_EXCL_STOP */
661 : }
662 0 : if (hash_size > HASH_MAX) {
663 : /* LCOV_EXCL_START */
664 : log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
665 : exit(EXIT_FAILURE);
666 : /* LCOV_EXCL_STOP */
667 : }
668 : /* check if it's a power of 2 */
669 0 : if ((hash_size & (hash_size - 1)) != 0) {
670 : /* LCOV_EXCL_START */
671 : log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
672 : exit(EXIT_FAILURE);
673 : /* LCOV_EXCL_STOP */
674 : }
675 :
676 0 : BLOCK_HASH_SIZE = hash_size;
677 6170 : } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
678 : char device[PATH_MAX];
679 : char* split_map[SPLIT_MAX + 1];
680 : unsigned split_mac;
681 : char* slash;
682 : uint64_t dev;
683 : int skip_access;
684 :
685 1284 : if (state->parity[level].split_mac != 0) {
686 : /* LCOV_EXCL_START */
687 : log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
688 : exit(EXIT_FAILURE);
689 : /* LCOV_EXCL_STOP */
690 : }
691 :
692 1284 : ret = sgetlasttok(f, buffer, sizeof(buffer));
693 1284 : if (ret < 0) {
694 : /* LCOV_EXCL_START */
695 : log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
696 : exit(EXIT_FAILURE);
697 : /* LCOV_EXCL_STOP */
698 : }
699 :
700 1284 : if (!*buffer) {
701 : /* LCOV_EXCL_START */
702 : log_fatal("Empty '%s' specification in '%s' at line %u\n", tag, path, line);
703 : exit(EXIT_FAILURE);
704 : /* LCOV_EXCL_STOP */
705 : }
706 :
707 1284 : split_mac = strsplit(split_map, SPLIT_MAX + 1, buffer, ",");
708 :
709 1284 : if (split_mac > SPLIT_MAX) {
710 : /* LCOV_EXCL_START */
711 : log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
712 : exit(EXIT_FAILURE);
713 : /* LCOV_EXCL_STOP */
714 : }
715 :
716 1284 : skip_access = 0;
717 1284 : state->parity[level].split_mac = split_mac;
718 6416 : for (s = 0; s < split_mac; ++s) {
719 5133 : pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
720 :
721 5133 : if (!state->opt.skip_parity_access) {
722 : struct stat st;
723 :
724 : /* get the device of the directory containing the parity file */
725 4421 : pathimport(device, sizeof(device), split_map[s]);
726 4421 : slash = strrchr(device, '/');
727 4421 : if (slash)
728 4421 : *slash = 0;
729 : else
730 0 : pathcpy(device, sizeof(device), ".");
731 :
732 4421 : if (stat(device, &st) == 0) {
733 4416 : dev = st.st_dev;
734 : } else {
735 : /* if the disk can be skipped */
736 5 : if (state->opt.force_device) {
737 : /* use a fake device, and mark the disk to be skipped */
738 4 : dev = 0;
739 4 : skip_access = 1;
740 4 : log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
741 : } else {
742 : /* LCOV_EXCL_START */
743 : log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
744 :
745 : /* in "fix" we allow to continue anyway */
746 : if (strcmp(state->command, "fix") == 0) {
747 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
748 : }
749 : exit(EXIT_FAILURE);
750 : /* LCOV_EXCL_STOP */
751 : }
752 : }
753 : } else {
754 : /* use a fake device */
755 712 : dev = 0;
756 : }
757 :
758 5132 : state->parity[level].split_map[s].device = dev;
759 : }
760 :
761 : /* store the global parity skip_access */
762 1283 : state->parity[level].skip_access = skip_access;
763 :
764 : /* adjust the level */
765 1283 : if (state->level < level + 1)
766 1015 : state->level = level + 1;
767 4886 : } else if (strcmp(tag, "share") == 0) {
768 41 : if (*state->share) {
769 : /* LCOV_EXCL_START */
770 : log_fatal("Multiple 'share' specification in '%s' at line %u\n", path, line);
771 : exit(EXIT_FAILURE);
772 : /* LCOV_EXCL_STOP */
773 : }
774 :
775 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
776 41 : if (ret < 0) {
777 : /* LCOV_EXCL_START */
778 : log_fatal("Invalid 'share' specification in '%s' at line %u\n", path, line);
779 : exit(EXIT_FAILURE);
780 : /* LCOV_EXCL_STOP */
781 : }
782 :
783 41 : if (!*buffer) {
784 : /* LCOV_EXCL_START */
785 : log_fatal("Empty 'share' specification in '%s' at line %u\n", path, line);
786 : exit(EXIT_FAILURE);
787 : /* LCOV_EXCL_STOP */
788 : }
789 :
790 41 : pathimport(state->share, sizeof(state->share), buffer);
791 4845 : } else if (strcmp(tag, "pool") == 0) {
792 : struct stat st;
793 :
794 41 : if (*state->pool) {
795 : /* LCOV_EXCL_START */
796 : log_fatal("Multiple 'pool' specification in '%s' at line %u\n", path, line);
797 : exit(EXIT_FAILURE);
798 : /* LCOV_EXCL_STOP */
799 : }
800 :
801 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
802 41 : if (ret < 0) {
803 : /* LCOV_EXCL_START */
804 : log_fatal("Invalid 'pool' specification in '%s' at line %u\n", path, line);
805 : exit(EXIT_FAILURE);
806 : /* LCOV_EXCL_STOP */
807 : }
808 :
809 41 : if (!*buffer) {
810 : /* LCOV_EXCL_START */
811 : log_fatal("Empty 'pool' specification in '%s' at line %u\n", path, line);
812 : exit(EXIT_FAILURE);
813 : /* LCOV_EXCL_STOP */
814 : }
815 :
816 41 : pathimport(state->pool, sizeof(state->pool), buffer);
817 :
818 : /* get the device of the directory containing the pool tree */
819 41 : if (stat(buffer, &st) != 0) {
820 : /* LCOV_EXCL_START */
821 : log_fatal("Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
822 : exit(EXIT_FAILURE);
823 : /* LCOV_EXCL_STOP */
824 : }
825 :
826 41 : state->pool_device = st.st_dev;
827 4804 : } else if (strcmp(tag, "content") == 0) {
828 : struct snapraid_content* content;
829 : char device[PATH_MAX];
830 : char* slash;
831 : struct stat st;
832 : uint64_t dev;
833 :
834 1545 : ret = sgetlasttok(f, buffer, sizeof(buffer));
835 1545 : if (ret < 0) {
836 : /* LCOV_EXCL_START */
837 : log_fatal("Invalid 'content' specification in '%s' at line %u\n", path, line);
838 : exit(EXIT_FAILURE);
839 : /* LCOV_EXCL_STOP */
840 : }
841 :
842 1545 : if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
843 : /* LCOV_EXCL_START */
844 : log_fatal("You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
845 : exit(EXIT_FAILURE);
846 : /* LCOV_EXCL_STOP */
847 : }
848 :
849 1545 : if (!*buffer) {
850 : /* LCOV_EXCL_START */
851 : log_fatal("Empty 'content' specification in '%s' at line %u\n", path, line);
852 : exit(EXIT_FAILURE);
853 : /* LCOV_EXCL_STOP */
854 : }
855 :
856 : /* check if the content file is already specified */
857 5780 : for (i = state->contentlist; i != 0; i = i->next) {
858 4235 : content = i->data;
859 4235 : if (pathcmp(content->content, buffer) == 0)
860 0 : break;
861 : }
862 1545 : if (i) {
863 : /* LCOV_EXCL_START */
864 : log_fatal("Duplicate 'content' specification in '%s' at line %u\n", path, line);
865 : exit(EXIT_FAILURE);
866 : /* LCOV_EXCL_STOP */
867 : }
868 :
869 : /* get the device of the directory containing the content file */
870 1545 : pathimport(device, sizeof(device), buffer);
871 1545 : slash = strrchr(device, '/');
872 1545 : if (slash)
873 1545 : *(slash + 1) = 0;
874 : else
875 0 : pathcpy(device, sizeof(device), ".");
876 1545 : if (stat(device, &st) == 0) {
877 1544 : dev = st.st_dev;
878 : } else {
879 1 : if (state->opt.skip_content_access) {
880 : /* use a fake device */
881 1 : dev = 0;
882 1 : log_fatal("WARNING! Skipping inaccessible content file '%s'...\n", buffer);
883 : } else {
884 : /* LCOV_EXCL_START */
885 : log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
886 : exit(EXIT_FAILURE);
887 : /* LCOV_EXCL_STOP */
888 : }
889 : }
890 :
891 : /* set the lock file at the first accessible content file */
892 1545 : if (state->lockfile[0] == 0 && dev != 0) {
893 267 : pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
894 267 : pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
895 : }
896 :
897 1545 : content = content_alloc(buffer, dev);
898 :
899 1545 : tommy_list_insert_tail(&state->contentlist, &content->node, content);
900 4856 : } else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
901 : /* "disk" is the deprecated name up to SnapRAID 9.x */
902 : char dir[PATH_MAX];
903 : char device[PATH_MAX];
904 : char uuid[UUID_MAX];
905 : struct snapraid_disk* disk;
906 : uint64_t dev;
907 : int skip_access;
908 :
909 1597 : ret = sgettok(f, buffer, sizeof(buffer));
910 1597 : if (ret < 0) {
911 : /* LCOV_EXCL_START */
912 : log_fatal("Invalid 'data' name specification in '%s' at line %u\n", path, line);
913 : exit(EXIT_FAILURE);
914 : /* LCOV_EXCL_STOP */
915 : }
916 :
917 1597 : if (!*buffer) {
918 : /* LCOV_EXCL_START */
919 : log_fatal("Empty 'data' name specification in '%s' at line %u\n", path, line);
920 : exit(EXIT_FAILURE);
921 : /* LCOV_EXCL_STOP */
922 : }
923 :
924 1597 : sgetspace(f);
925 :
926 1597 : ret = sgetlasttok(f, dir, sizeof(dir));
927 1597 : if (ret < 0) {
928 : /* LCOV_EXCL_START */
929 : log_fatal("Invalid 'data' dir specification in '%s' at line %u\n", path, line);
930 : exit(EXIT_FAILURE);
931 : /* LCOV_EXCL_STOP */
932 : }
933 :
934 1597 : if (!*dir) {
935 : /* LCOV_EXCL_START */
936 : log_fatal("Empty 'data' dir specification in '%s' at line %u\n", path, line);
937 : exit(EXIT_FAILURE);
938 : /* LCOV_EXCL_STOP */
939 : }
940 :
941 : /* get the device of the dir */
942 1597 : pathimport(device, sizeof(device), dir);
943 :
944 : /* check if the disk name already exists */
945 5577 : for (i = state->disklist; i != 0; i = i->next) {
946 3980 : disk = i->data;
947 3980 : if (strcmp(disk->name, buffer) == 0)
948 0 : break;
949 : }
950 1597 : if (i) {
951 : /* LCOV_EXCL_START */
952 : log_fatal("Duplicate 'data' name '%s' at line %u\n", buffer, line);
953 : exit(EXIT_FAILURE);
954 : /* LCOV_EXCL_STOP */
955 : }
956 :
957 : /* if the disk has to be present */
958 1597 : skip_access = 0;
959 1597 : if (!state->opt.skip_disk_access) {
960 : struct stat st;
961 :
962 1441 : if (stat(device, &st) == 0) {
963 1440 : dev = st.st_dev;
964 :
965 : /* read the uuid, if unsupported use an empty one */
966 1440 : if (devuuid(dev, dir, uuid, sizeof(uuid)) != 0) {
967 0 : *uuid = 0;
968 : }
969 :
970 : /* fake a different UUID when testing */
971 1440 : if (state->opt.fake_uuid) {
972 2 : snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
973 2 : --state->opt.fake_uuid;
974 : }
975 : } else {
976 : /* if the disk can be skipped */
977 1 : if (state->opt.force_device) {
978 : /* use a fake device, and mark the disk to be skipped */
979 1 : dev = 0;
980 1 : *uuid = 0;
981 1 : skip_access = 1;
982 1 : log_fatal("DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
983 : } else {
984 : /* LCOV_EXCL_START */
985 : log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
986 :
987 : /* in "fix" we allow to continue anyway */
988 : if (strcmp(state->command, "fix") == 0) {
989 : log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
990 : }
991 : exit(EXIT_FAILURE);
992 : /* LCOV_EXCL_STOP */
993 : }
994 : }
995 : } else {
996 : /* use a fake device */
997 156 : dev = 0;
998 156 : *uuid = 0;
999 : }
1000 :
1001 1597 : disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
1002 :
1003 1597 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1004 1662 : } else if (strcmp(tag, "smartctl") == 0) {
1005 : char custom[PATH_MAX];
1006 :
1007 368 : ret = sgettok(f, buffer, sizeof(buffer));
1008 368 : if (ret < 0) {
1009 : /* LCOV_EXCL_START */
1010 : log_fatal("Invalid 'smartctl' name in '%s' at line %u\n", path, line);
1011 : exit(EXIT_FAILURE);
1012 : /* LCOV_EXCL_STOP */
1013 : }
1014 :
1015 368 : if (!*buffer) {
1016 : /* LCOV_EXCL_START */
1017 : log_fatal("Empty 'smartctl' name in '%s' at line %u\n", path, line);
1018 : exit(EXIT_FAILURE);
1019 : /* LCOV_EXCL_STOP */
1020 : }
1021 :
1022 368 : sgetspace(f);
1023 :
1024 368 : ret = sgetlasttok(f, custom, sizeof(custom));
1025 368 : if (ret < 0) {
1026 : /* LCOV_EXCL_START */
1027 : log_fatal("Invalid 'smartctl' option in '%s' at line %u\n", path, line);
1028 : exit(EXIT_FAILURE);
1029 : /* LCOV_EXCL_STOP */
1030 : }
1031 :
1032 368 : if (!*custom) {
1033 : /* LCOV_EXCL_START */
1034 : log_fatal("Empty 'smartctl' option in '%s' at line %u\n", path, line);
1035 : exit(EXIT_FAILURE);
1036 : /* LCOV_EXCL_STOP */
1037 : }
1038 :
1039 368 : if (validate_smartctl(custom) != 0) {
1040 : /* LCOV_EXCL_START */
1041 : log_fatal("Invalid 'smartctl' option in '%s' at line %u\n", path, line);
1042 : exit(EXIT_FAILURE);
1043 : /* LCOV_EXCL_STOP */
1044 : }
1045 :
1046 : /* search for parity */
1047 368 : if (lev_config_scan(buffer, &level, 0) == 0) {
1048 184 : if (state->parity[level].smartctl[0] != 0) {
1049 : /* LCOV_EXCL_START */
1050 : log_fatal("Duplicate parity 'smartctl' '%s' at line %u\n", buffer, line);
1051 : exit(EXIT_FAILURE);
1052 : /* LCOV_EXCL_STOP */
1053 : }
1054 :
1055 184 : pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
1056 : } else {
1057 : struct snapraid_disk* disk;
1058 :
1059 : /* search the disk */
1060 184 : disk = 0;
1061 184 : for (i = state->disklist; i != 0; i = i->next) {
1062 184 : disk = i->data;
1063 184 : if (strcmp(disk->name, buffer) == 0)
1064 184 : break;
1065 : }
1066 184 : if (!disk) {
1067 : /* LCOV_EXCL_START */
1068 : log_fatal("Missing disk 'smartctl' '%s' at line %u\n", buffer, line);
1069 : exit(EXIT_FAILURE);
1070 : /* LCOV_EXCL_STOP */
1071 : }
1072 184 : if (disk->smartctl[0] != 0) {
1073 : /* LCOV_EXCL_START */
1074 : log_fatal("Duplicate disk name '%s' at line %u\n", buffer, line);
1075 : exit(EXIT_FAILURE);
1076 : /* LCOV_EXCL_STOP */
1077 : }
1078 :
1079 184 : pathcpy(disk->smartctl, sizeof(disk->smartctl), custom);
1080 : }
1081 1294 : } else if (strcmp(tag, "smartignore") == 0) {
1082 : int* smart;
1083 :
1084 366 : ret = sgettok(f, buffer, sizeof(buffer));
1085 366 : if (ret < 0) {
1086 : /* LCOV_EXCL_START */
1087 : log_fatal("Invalid 'smartignore' name in '%s' at line %u\n", path, line);
1088 : exit(EXIT_FAILURE);
1089 : /* LCOV_EXCL_STOP */
1090 : }
1091 :
1092 366 : if (!*buffer) {
1093 : /* LCOV_EXCL_START */
1094 : log_fatal("Empty 'smartignore' name in '%s' at line %u\n", path, line);
1095 : exit(EXIT_FAILURE);
1096 : /* LCOV_EXCL_STOP */
1097 : }
1098 :
1099 366 : sgetspace(f);
1100 :
1101 366 : if (strcmp(buffer, "*") == 0) {
1102 183 : smart = state->smartignore;
1103 183 : } else if (lev_config_scan(buffer, &level, 0) == 0) { /* search for parity */
1104 183 : smart = state->parity[level].smartignore;
1105 : } else {
1106 : struct snapraid_disk* disk;
1107 :
1108 : /* search the disk */
1109 0 : disk = 0;
1110 0 : for (i = state->disklist; i != 0; i = i->next) {
1111 0 : disk = i->data;
1112 0 : if (strcmp(disk->name, buffer) == 0)
1113 0 : break;
1114 : }
1115 0 : if (!disk) {
1116 : /* LCOV_EXCL_START */
1117 : log_fatal("Missing disk 'smartctl' '%s' at line %u\n", buffer, line);
1118 : exit(EXIT_FAILURE);
1119 : /* LCOV_EXCL_STOP */
1120 : }
1121 :
1122 0 : smart = disk->smartignore;
1123 : }
1124 :
1125 366 : int si = 0;
1126 :
1127 366 : while (1) {
1128 : char* e;
1129 :
1130 732 : ret = sgettok(f, buffer, sizeof(buffer));
1131 732 : if (ret < 0) {
1132 : /* LCOV_EXCL_START */
1133 : log_fatal("Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1134 : exit(EXIT_FAILURE);
1135 : /* LCOV_EXCL_STOP */
1136 : }
1137 :
1138 732 : if (!*buffer)
1139 366 : break;
1140 :
1141 366 : if (si == SMART_IGNORE_MAX) {
1142 : /* LCOV_EXCL_START */
1143 : log_fatal("Too many 'smartignore' specification in '%s' at line %u\n", path, line);
1144 : exit(EXIT_FAILURE);
1145 : /* LCOV_EXCL_STOP */
1146 : }
1147 :
1148 366 : smart[si] = strtoul(buffer, &e, 0);
1149 :
1150 366 : if (!e || *e) {
1151 : /* LCOV_EXCL_START */
1152 : log_fatal("Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1153 : exit(EXIT_FAILURE);
1154 : /* LCOV_EXCL_STOP */
1155 : }
1156 :
1157 366 : ++si;
1158 :
1159 366 : sgetspace(f);
1160 : }
1161 :
1162 366 : if (si == 0) {
1163 : /* LCOV_EXCL_START */
1164 : log_fatal("Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
1165 : exit(EXIT_FAILURE);
1166 : /* LCOV_EXCL_STOP */
1167 : }
1168 928 : } else if (strcmp(tag, "temp_limit") == 0) {
1169 : unsigned temp;
1170 :
1171 1 : ret = sgetu32(f, &temp);
1172 1 : if (ret < 0) {
1173 : /* LCOV_EXCL_START */
1174 : log_fatal("Invalid 'temp_limit' specification in '%s' at line %u\n", path, line);
1175 : exit(EXIT_FAILURE);
1176 : /* LCOV_EXCL_STOP */
1177 : }
1178 1 : if (temp < 40 || temp > 70) {
1179 : /* LCOV_EXCL_START */
1180 : log_fatal("Invalid 'temp_limit' temperature specification in '%s' at line %u. It must be between 40 and 70\n", path, line);
1181 : exit(EXIT_FAILURE);
1182 : /* LCOV_EXCL_STOP */
1183 : }
1184 :
1185 1 : state->thermal_temperature_limit = temp;
1186 927 : } else if (strcmp(tag, "temp_sleep") == 0) {
1187 : unsigned time;
1188 :
1189 1 : ret = sgetu32(f, &time);
1190 1 : if (ret < 0) {
1191 : /* LCOV_EXCL_START */
1192 : log_fatal("Invalid 'temp_sleep' specification in '%s' at line %u\n", path, line);
1193 : exit(EXIT_FAILURE);
1194 : /* LCOV_EXCL_STOP */
1195 : }
1196 1 : if (time < 3 || time > 120) {
1197 : /* LCOV_EXCL_START */
1198 : log_fatal("Invalid 'temp_sleep' temperature specification in '%s' at line %u. It must be between 3 and 120\n", path, line);
1199 : exit(EXIT_FAILURE);
1200 : /* LCOV_EXCL_STOP */
1201 : }
1202 :
1203 1 : state->thermal_cooldown_time = time * 60;
1204 926 : } else if (strcmp(tag, "nohidden") == 0) {
1205 0 : state->filter_hidden = 1;
1206 926 : } else if (strcmp(tag, "exclude") == 0) {
1207 : struct snapraid_filter* filter;
1208 :
1209 267 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1210 267 : if (ret < 0) {
1211 : /* LCOV_EXCL_START */
1212 : log_fatal("Invalid 'exclude' specification in '%s' at line %u\n", path, line);
1213 : exit(EXIT_FAILURE);
1214 : /* LCOV_EXCL_STOP */
1215 : }
1216 :
1217 267 : if (!*buffer) {
1218 : /* LCOV_EXCL_START */
1219 : log_fatal("Empty 'exclude' specification in '%s' at line %u\n", path, line);
1220 : exit(EXIT_FAILURE);
1221 : /* LCOV_EXCL_STOP */
1222 : }
1223 :
1224 267 : filter = filter_alloc_file(-1, buffer);
1225 267 : if (!filter) {
1226 : /* LCOV_EXCL_START */
1227 : log_fatal("Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
1228 : log_fatal("Filters with relative paths are not supported. Ensure to add an initial slash\n");
1229 : exit(EXIT_FAILURE);
1230 : /* LCOV_EXCL_STOP */
1231 : }
1232 267 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1233 659 : } else if (strcmp(tag, "include") == 0) {
1234 : struct snapraid_filter* filter;
1235 :
1236 267 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1237 267 : if (ret < 0) {
1238 : /* LCOV_EXCL_START */
1239 : log_fatal("Invalid 'include' specification in '%s' at line %u\n", path, line);
1240 : exit(EXIT_FAILURE);
1241 : /* LCOV_EXCL_STOP */
1242 : }
1243 :
1244 267 : if (!*buffer) {
1245 : /* LCOV_EXCL_START */
1246 : log_fatal("Empty 'include' specification in '%s' at line %u\n", path, line);
1247 : exit(EXIT_FAILURE);
1248 : /* LCOV_EXCL_STOP */
1249 : }
1250 :
1251 267 : filter = filter_alloc_file(1, buffer);
1252 267 : if (!filter) {
1253 : /* LCOV_EXCL_START */
1254 : log_fatal("Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
1255 : log_fatal("Filters with relative paths are not supported. Ensure to add an initial slash\n");
1256 : exit(EXIT_FAILURE);
1257 : /* LCOV_EXCL_STOP */
1258 : }
1259 267 : tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
1260 392 : } else if (strcmp(tag, "autosave") == 0) {
1261 : char* e;
1262 :
1263 41 : ret = sgetlasttok(f, buffer, sizeof(buffer));
1264 41 : if (ret < 0) {
1265 : /* LCOV_EXCL_START */
1266 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1267 : exit(EXIT_FAILURE);
1268 : /* LCOV_EXCL_STOP */
1269 : }
1270 :
1271 41 : if (!*buffer) {
1272 : /* LCOV_EXCL_START */
1273 : log_fatal("Empty 'autosave' specification in '%s' at line %u\n", path, line);
1274 : exit(EXIT_FAILURE);
1275 : /* LCOV_EXCL_STOP */
1276 : }
1277 :
1278 41 : state->autosave = strtoul(buffer, &e, 0);
1279 :
1280 41 : if (!e || *e) {
1281 : /* LCOV_EXCL_START */
1282 : log_fatal("Invalid 'autosave' specification in '%s' at line %u\n", path, line);
1283 : exit(EXIT_FAILURE);
1284 : /* LCOV_EXCL_STOP */
1285 : }
1286 :
1287 : /* convert to GB */
1288 41 : state->autosave *= GIGA;
1289 351 : } else if (tag[0] == 0) {
1290 : /* allow empty lines */
1291 41 : } else if (tag[0] == '#') {
1292 41 : ret = sgetline(f, buffer, sizeof(buffer));
1293 41 : if (ret < 0) {
1294 : /* LCOV_EXCL_START */
1295 : log_fatal("Invalid comment in '%s' at line %u\n", path, line);
1296 : exit(EXIT_FAILURE);
1297 : /* LCOV_EXCL_STOP */
1298 : }
1299 : } else {
1300 : /* LCOV_EXCL_START */
1301 : log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line);
1302 : exit(EXIT_FAILURE);
1303 : /* LCOV_EXCL_STOP */
1304 : }
1305 :
1306 : /* skip final spaces */
1307 6437 : sgetspace(f);
1308 :
1309 : /* next line */
1310 6437 : c = sgeteol(f);
1311 6437 : if (c == EOF) {
1312 267 : break;
1313 : }
1314 6170 : if (c != '\n') {
1315 : /* LCOV_EXCL_START */
1316 : log_fatal("Extra data in '%s' at line %u\n", path, line);
1317 : exit(EXIT_FAILURE);
1318 : /* LCOV_EXCL_STOP */
1319 : }
1320 6170 : ++line;
1321 : }
1322 :
1323 267 : if (serror(f)) {
1324 : /* LCOV_EXCL_START */
1325 : log_fatal("Error reading the configuration file '%s' at line %u\n", path, line);
1326 : exit(EXIT_FAILURE);
1327 : /* LCOV_EXCL_STOP */
1328 : }
1329 :
1330 267 : sclose(f);
1331 :
1332 267 : state_config_check(state, path, filterlist_disk);
1333 :
1334 : /* select the default hash */
1335 267 : if (state->opt.force_murmur3) {
1336 1 : state->besthash = HASH_MURMUR3;
1337 266 : } else if (state->opt.force_spooky2) {
1338 1 : state->besthash = HASH_SPOOKY2;
1339 : } else {
1340 : #ifdef CONFIG_X86
1341 : if (sizeof(void*) == 4 && !raid_cpu_has_slowmult())
1342 : state->besthash = HASH_MURMUR3;
1343 : else
1344 265 : state->besthash = HASH_SPOOKY2;
1345 : #else
1346 : if (sizeof(void*) == 4)
1347 : state->besthash = HASH_MURMUR3;
1348 : else
1349 : state->besthash = HASH_SPOOKY2;
1350 : #endif
1351 : }
1352 :
1353 : /* by default use the best hash */
1354 267 : state->hash = state->besthash;
1355 :
1356 : /* by default use a random hash seed */
1357 267 : if (randomize(state->hashseed, HASH_MAX) != 0) {
1358 : /* LCOV_EXCL_START */
1359 : log_fatal("Failed to retrieve random values.\n");
1360 : exit(EXIT_FAILURE);
1361 : /* LCOV_EXCL_STOP */
1362 : }
1363 :
1364 : /* no previous hash by default */
1365 267 : state->prevhash = HASH_UNDEFINED;
1366 :
1367 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
1368 :
1369 267 : log_tag("blocksize:%u\n", state->block_size);
1370 1864 : for (i = state->disklist; i != 0; i = i->next) {
1371 1597 : struct snapraid_disk* disk = i->data;
1372 1597 : log_tag("data:%s:%s\n", disk->name, disk->dir);
1373 : }
1374 :
1375 267 : log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
1376 1545 : for (l = 0; l < state->level; ++l)
1377 6390 : for (s = 0; s < state->parity[l].split_mac; ++s)
1378 5112 : log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
1379 267 : if (state->pool[0] != 0)
1380 41 : log_tag("pool:%s\n", state->pool);
1381 267 : if (state->share[0] != 0)
1382 41 : log_tag("share:%s\n", state->share);
1383 267 : if (state->autosave != 0)
1384 41 : log_tag("autosave:%" PRIu64 "\n", state->autosave);
1385 801 : for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
1386 : char out[PATH_MAX];
1387 534 : struct snapraid_filter* filter = i->data;
1388 534 : log_tag("filter:%s\n", filter_type(filter, out, sizeof(out)));
1389 : }
1390 267 : if (state->filter_hidden)
1391 0 : log_tag("filter:nohidden:\n");
1392 267 : log_flush();
1393 : }
1394 :
1395 : /**
1396 : * Find a disk by name.
1397 : */
1398 7005 : static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
1399 : {
1400 : tommy_node* i;
1401 :
1402 24441 : for (i = state->disklist; i != 0; i = i->next) {
1403 24433 : struct snapraid_disk* disk = i->data;
1404 24433 : if (strcmp(disk->name, name) == 0)
1405 6997 : return disk;
1406 : }
1407 :
1408 8 : if (state->no_conf) {
1409 : /* without a configuration file, add disks automatically */
1410 : struct snapraid_disk* disk;
1411 :
1412 6 : disk = disk_alloc(name, "DUMMY/", -1, "", 0);
1413 :
1414 6 : tommy_list_insert_tail(&state->disklist, &disk->node, disk);
1415 :
1416 6 : return disk;
1417 : }
1418 :
1419 2 : return 0;
1420 : }
1421 :
1422 : /**
1423 : * Find a disk by UUID.
1424 : */
1425 2 : static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
1426 : {
1427 : tommy_node* i;
1428 2 : struct snapraid_disk* found = 0;
1429 :
1430 : /* special test case to find the first matching UUID */
1431 : /* when testing UUID are all equal or not supported */
1432 : /* and we should handle this case specifically */
1433 2 : if (state->opt.match_first_uuid)
1434 2 : return state->disklist->data;
1435 :
1436 : /* LCOV_EXCL_START */
1437 : /* never find an empty uuid */
1438 : if (!*uuid)
1439 : return 0;
1440 :
1441 : for (i = state->disklist; i != 0; i = i->next) {
1442 : struct snapraid_disk* disk = i->data;
1443 : if (strcmp(disk->uuid, uuid) == 0) {
1444 : /* never match duplicate UUID */
1445 : if (found)
1446 : return 0;
1447 : found = disk;
1448 : }
1449 : }
1450 :
1451 : return found;
1452 : /* LCOV_EXCL_STOP */
1453 : }
1454 :
1455 : /**
1456 : * Update the disk mapping if required.
1457 : */
1458 261 : static void state_map(struct snapraid_state* state)
1459 : {
1460 : unsigned hole;
1461 : tommy_node* i;
1462 : unsigned uuid_mismatch;
1463 : unsigned diskcount;
1464 : unsigned l, s;
1465 :
1466 : /* remove all the mapping without a disk */
1467 : /* this happens when a disk is removed from the configuration file */
1468 : /* From SnapRAID 4.0 mappings are automatically removed if a disk is not used */
1469 : /* when saving the content file, but we keep this code to import older content files. */
1470 1664 : for (i = state->maplist; i != 0; ) {
1471 1403 : struct snapraid_map* map = i->data;
1472 : struct snapraid_disk* disk;
1473 :
1474 1403 : disk = find_disk_by_name(state, map->name);
1475 :
1476 : /* go to the next mapping before removing */
1477 1403 : i = i->next;
1478 :
1479 1403 : if (disk == 0) {
1480 : /* disk not found, remove the mapping */
1481 0 : tommy_list_remove_existing(&state->maplist, &map->node);
1482 0 : map_free(map);
1483 : }
1484 : }
1485 :
1486 : /* maps each unmapped disk present in the configuration file in the first available hole */
1487 : /* this happens when you add disks for the first time in the configuration file */
1488 261 : hole = 0; /* first position to try */
1489 1822 : for (i = state->disklist; i != 0; i = i->next) {
1490 1561 : struct snapraid_disk* disk = i->data;
1491 : struct snapraid_map* map;
1492 : tommy_node* j;
1493 :
1494 : /* check if the disk is already mapped */
1495 5451 : for (j = state->maplist; j != 0; j = j->next) {
1496 5293 : map = j->data;
1497 5293 : if (strcmp(disk->name, map->name) == 0) {
1498 : /* mapping found */
1499 1403 : break;
1500 : }
1501 : }
1502 1561 : if (j != 0) {
1503 : /* mapping is present, then copy the free blocks into to disk */
1504 1403 : disk->total_blocks = map->total_blocks;
1505 1403 : disk->free_blocks = map->free_blocks;
1506 1403 : continue;
1507 : }
1508 :
1509 : /* mapping not found, search for an hole */
1510 : while (1) {
1511 1100 : for (j = state->maplist; j != 0; j = j->next) {
1512 942 : map = j->data;
1513 942 : if (map->position == hole) {
1514 : /* position already used */
1515 158 : break;
1516 : }
1517 : }
1518 316 : if (j == 0) {
1519 : /* hole found */
1520 158 : break;
1521 : }
1522 :
1523 : /* try with the next one */
1524 158 : ++hole;
1525 : }
1526 :
1527 : /* insert the new mapping */
1528 158 : map = map_alloc(disk->name, hole, 0, 0, "");
1529 :
1530 158 : tommy_list_insert_tail(&state->maplist, &map->node, map);
1531 : }
1532 :
1533 : /* without configuration don't check for number of data disks or uuid changes */
1534 261 : if (state->no_conf)
1535 1 : return;
1536 :
1537 : /* counter for the number of UUID mismatches */
1538 260 : uuid_mismatch = 0;
1539 :
1540 : /* check if mapping match the disk uuid */
1541 260 : if (!state->opt.skip_disk_access) {
1542 1633 : for (i = state->maplist; i != 0; i = i->next) {
1543 1399 : struct snapraid_map* map = i->data;
1544 : struct snapraid_disk* disk;
1545 :
1546 1399 : disk = find_disk_by_name(state, map->name);
1547 1399 : if (disk == 0) {
1548 : /* LCOV_EXCL_START */
1549 : log_fatal("Internal inconsistency: Mapping not found for '%s'\n", map->name);
1550 : os_abort();
1551 : /* LCOV_EXCL_STOP */
1552 : }
1553 :
1554 1399 : if (disk->has_unsupported_uuid) {
1555 : /* if uuid is not available, skip this one */
1556 1 : continue;
1557 : }
1558 :
1559 : /* if the uuid is changed */
1560 1398 : if (strcmp(disk->uuid, map->uuid) != 0) {
1561 : /* mark the disk as with an UUID change */
1562 113 : disk->has_different_uuid = 1;
1563 :
1564 : /* if the previous uuid is available */
1565 113 : if (map->uuid[0] != 0) {
1566 : /* count the number of uuid change */
1567 4 : ++uuid_mismatch;
1568 4 : log_fatal("UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
1569 : } else {
1570 : /* no message here, because having a disk without */
1571 : /* UUID is the normal state of an empty disk */
1572 109 : disk->had_empty_uuid = 1;
1573 : }
1574 :
1575 : /* update the uuid in the mapping, */
1576 113 : pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
1577 :
1578 : /* write the new state with the new uuid */
1579 113 : state->need_write = 1;
1580 : }
1581 : }
1582 : }
1583 :
1584 : /* check the parity uuid */
1585 260 : if (!state->opt.skip_parity_access) {
1586 1280 : for (l = 0; l < state->level; ++l) {
1587 5290 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1588 : char uuid[UUID_MAX];
1589 : int ret;
1590 :
1591 4232 : ret = devuuid(state->parity[l].split_map[s].device, state->parity[l].split_map[s].path, uuid, sizeof(uuid));
1592 :
1593 4232 : if (ret != 0) {
1594 : /* uuid not available, just ignore */
1595 4 : continue;
1596 : }
1597 :
1598 : /* if the uuid is changed */
1599 4228 : if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
1600 : /* if the previous uuid is available */
1601 184 : if (state->parity[l].split_map[s].uuid[0] != 0) {
1602 : /* count the number of uuid change */
1603 0 : ++uuid_mismatch;
1604 0 : log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
1605 : }
1606 :
1607 : /* update the uuid */
1608 184 : pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
1609 :
1610 : /* write the new state with the new uuid */
1611 184 : state->need_write = 1;
1612 : }
1613 : }
1614 : }
1615 : }
1616 :
1617 260 : if (!state->opt.force_uuid && uuid_mismatch > state->level) {
1618 : /* LCOV_EXCL_START */
1619 : log_fatal("Too many disks have changed UUIDs since the last 'sync'.\n");
1620 : log_fatal("If this happened because you actually replaced them,\n");
1621 : log_fatal("you can still '%s', using 'snapraid --force-uuid %s'.\n", state->command, state->command);
1622 : log_fatal("Alternatively, you may have misconfigured the disk mount points,\n");
1623 : log_fatal("and you need to restore the mount points to the state of the latest sync.\n");
1624 : exit(EXIT_FAILURE);
1625 : /* LCOV_EXCL_STOP */
1626 : }
1627 :
1628 : /* count the number of data disks, including holes left after removing some */
1629 260 : diskcount = 0;
1630 1815 : for (i = state->maplist; i != 0; i = i->next) {
1631 1555 : struct snapraid_map* map = i->data;
1632 :
1633 1555 : if (map->position + 1 > diskcount)
1634 1373 : diskcount = map->position + 1;
1635 : }
1636 :
1637 : /* ensure to don't go over the limit of the RAID engine */
1638 260 : if (diskcount > RAID_DATA_MAX) {
1639 : /* LCOV_EXCL_START */
1640 : log_fatal("Too many data disks. Maximum allowed is %u.\n", RAID_DATA_MAX);
1641 : exit(EXIT_FAILURE);
1642 : /* LCOV_EXCL_STOP */
1643 : }
1644 :
1645 : /* now count the real number of data disks, excluding holes left after removing some */
1646 260 : diskcount = tommy_list_count(&state->maplist);
1647 :
1648 : /* recommend number of parities */
1649 260 : if (!state->opt.no_warnings) {
1650 : /* intentionally use log_fatal() instead of log_error() to give more visibility at the warning */
1651 1 : if (diskcount >= 36 && state->level < 6) {
1652 0 : log_fatal("WARNING! For %u disks, it's recommended to use six parity levels.\n", diskcount);
1653 1 : } else if (diskcount >= 29 && state->level < 5) {
1654 0 : log_fatal("WARNING! For %u disks, it's recommended to use five parity levels.\n", diskcount);
1655 1 : } else if (diskcount >= 22 && state->level < 4) {
1656 0 : log_fatal("WARNING! For %u disks, it's recommended to use four parity levels.\n", diskcount);
1657 1 : } else if (diskcount >= 15 && state->level < 3) {
1658 0 : log_fatal("WARNING! For %u disks, it's recommended to use three parity levels.\n", diskcount);
1659 1 : } else if (diskcount >= 5 && state->level < 2) {
1660 0 : log_fatal("WARNING! For %u disks, it's recommended to use two parity levels.\n", diskcount);
1661 : }
1662 : }
1663 : }
1664 :
1665 173 : void state_refresh(struct snapraid_state* state)
1666 : {
1667 : tommy_node* i;
1668 : unsigned l, s;
1669 :
1670 : /* for all disks */
1671 1209 : for (i = state->maplist; i != 0; i = i->next) {
1672 1036 : struct snapraid_map* map = i->data;
1673 : struct snapraid_disk* disk;
1674 : uint64_t total_space;
1675 : uint64_t free_space;
1676 : int ret;
1677 :
1678 1036 : disk = find_disk_by_name(state, map->name);
1679 1036 : if (disk == 0) {
1680 : /* LCOV_EXCL_START */
1681 : log_fatal("Internal inconsistency: Mapping not found for '%s'\n", map->name);
1682 : os_abort();
1683 : /* LCOV_EXCL_STOP */
1684 : }
1685 :
1686 1036 : ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space);
1687 1036 : if (ret != 0) {
1688 : /* LCOV_EXCL_START */
1689 : log_fatal("Error accessing disk '%s' to retrieve file-system info. %s.\n", disk->dir, strerror(errno));
1690 : exit(EXIT_FAILURE);
1691 : /* LCOV_EXCL_STOP */
1692 : }
1693 :
1694 : /* set the new free blocks */
1695 1036 : map->total_blocks = total_space / state->block_size;
1696 1036 : map->free_blocks = free_space / state->block_size;
1697 :
1698 : /* also update the disk info */
1699 1036 : disk->total_blocks = map->total_blocks;
1700 1036 : disk->free_blocks = map->free_blocks;
1701 : }
1702 :
1703 : /* for all parities */
1704 1141 : for (l = 0; l < state->level; ++l) {
1705 : /* set the new free blocks */
1706 968 : state->parity[l].total_blocks = 0;
1707 968 : state->parity[l].free_blocks = 0;
1708 :
1709 4840 : for (s = 0; s < state->parity[l].split_mac; ++s) {
1710 : uint64_t total_space;
1711 : uint64_t free_space;
1712 : int ret;
1713 :
1714 3872 : ret = fsinfo(state->parity[l].split_map[s].path, 0, 0, &total_space, &free_space);
1715 3872 : if (ret != 0) {
1716 : /* LCOV_EXCL_START */
1717 : log_fatal("Error accessing file '%s' to retrieve file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
1718 : exit(EXIT_FAILURE);
1719 : /* LCOV_EXCL_STOP */
1720 : }
1721 :
1722 : /* add the new free blocks */
1723 3872 : state->parity[l].total_blocks += total_space / state->block_size;
1724 3872 : state->parity[l].free_blocks += free_space / state->block_size;
1725 : }
1726 : }
1727 :
1728 : /* note what we don't set need_write = 1, because we don't want */
1729 : /* to update the content file only for the free space info. */
1730 173 : }
1731 :
1732 : /**
1733 : * Check the content.
1734 : */
1735 254 : static void state_content_check(struct snapraid_state* state, const char* path)
1736 : {
1737 : tommy_node* i;
1738 :
1739 : /* check that any map has different name and position */
1740 1773 : for (i = state->maplist; i != 0; i = i->next) {
1741 1519 : struct snapraid_map* map = i->data;
1742 : tommy_node* j;
1743 5304 : for (j = i->next; j != 0; j = j->next) {
1744 3785 : struct snapraid_map* other = j->data;
1745 3785 : if (strcmp(map->name, other->name) == 0) {
1746 : /* LCOV_EXCL_START */
1747 : log_fatal("Conflicting 'map' disk specification in '%s'\n", path);
1748 : exit(EXIT_FAILURE);
1749 : /* LCOV_EXCL_STOP */
1750 : }
1751 3785 : if (map->position == other->position) {
1752 : /* LCOV_EXCL_START */
1753 : log_fatal("Conflicting 'map' index specification in '%s'\n", path);
1754 : exit(EXIT_FAILURE);
1755 : /* LCOV_EXCL_STOP */
1756 : }
1757 : }
1758 : }
1759 254 : }
1760 :
1761 : /**
1762 : * Check if the position is REQUIRED, or we can completely clear it from the state.
1763 : *
1764 : * Note that position with only DELETED blocks are discharged.
1765 : */
1766 734936 : static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
1767 : {
1768 : tommy_node* i;
1769 :
1770 : /* check for each disk */
1771 780746 : for (i = state->disklist; i != 0; i = i->next) {
1772 780746 : struct snapraid_disk* disk = i->data;
1773 780746 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1774 :
1775 : /* if we have at least one file, the position is needed */
1776 780746 : if (block_has_file(block))
1777 734936 : return 1;
1778 : }
1779 :
1780 0 : return 0;
1781 : }
1782 :
1783 : /**
1784 : * Check if the info block is REQUIREQ.
1785 : *
1786 : * This is used to ensure that we keep the last check used for scrubbing.
1787 : * and that we add it when importing old context files.
1788 : *
1789 : * Note that you can have position without info blocks, for example
1790 : * if all the blocks are not synced.
1791 : *
1792 : * Note also that not requiring an info block, doesn't mean that if present it
1793 : * can be discarded.
1794 : */
1795 1550339 : static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
1796 : {
1797 : tommy_node* i;
1798 :
1799 : /* check for each disk */
1800 2050904 : for (i = state->disklist; i != 0; i = i->next) {
1801 1992139 : struct snapraid_disk* disk = i->data;
1802 1992139 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1803 :
1804 : /* if we have at least one synced file, the info is required */
1805 1992139 : if (block_state_get(block) == BLOCK_STATE_BLK)
1806 1491574 : return 1;
1807 : }
1808 :
1809 58765 : return 0;
1810 : }
1811 :
1812 0 : static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
1813 : {
1814 : tommy_node* i;
1815 :
1816 : /* check for each disk if block is really used */
1817 0 : for (i = state->disklist; i != 0; i = i->next) {
1818 0 : struct snapraid_disk* disk = i->data;
1819 0 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1820 :
1821 : /* if the block is deleted */
1822 0 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1823 : /* set it to empty */
1824 0 : fs_deallocate(disk, pos);
1825 : }
1826 : }
1827 0 : }
1828 :
1829 : /**
1830 : * Check if a block position in a disk is deleted.
1831 : */
1832 4402643 : static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
1833 : {
1834 4402643 : struct snapraid_block* block = fs_par2block_find(disk, pos);
1835 :
1836 4402643 : return block_state_get(block) == BLOCK_STATE_DELETED;
1837 : }
1838 :
1839 : /**
1840 : * Flush the file checking the final CRC.
1841 : * We exploit the fact that the CRC is always stored in the last 4 bytes.
1842 : */
1843 1 : static void decoding_error(const char* path, STREAM* f)
1844 : {
1845 : unsigned char buf[4];
1846 : uint32_t crc_stored;
1847 : uint32_t crc_computed;
1848 :
1849 1 : if (seof(f)) {
1850 : /* LCOV_EXCL_START */
1851 : log_fatal("Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1852 : log_fatal("This content file is truncated. Please use an alternate copy.\n");
1853 : exit(EXIT_FAILURE);
1854 : /* LCOV_EXCL_STOP */
1855 : }
1856 :
1857 1 : if (serror(f)) {
1858 : /* LCOV_EXCL_START */
1859 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1860 : exit(EXIT_FAILURE);
1861 : /* LCOV_EXCL_STOP */
1862 : }
1863 :
1864 1 : log_fatal("Error decoding '%s' at offset %" PRIi64 "\n", path, stell(f));
1865 :
1866 1 : if (sdeplete(f, buf) != 0) {
1867 : /* LCOV_EXCL_START */
1868 : log_fatal("Failed to flush content file '%s' at offset %" PRIi64 "\n", path, stell(f));
1869 : exit(EXIT_FAILURE);
1870 : /* LCOV_EXCL_STOP */
1871 : }
1872 :
1873 : /* get the stored crc from the last four bytes */
1874 1 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
1875 :
1876 : /* get the computed crc */
1877 1 : crc_computed = scrc(f);
1878 :
1879 : /* adjust the stored crc to include itself */
1880 1 : crc_stored = crc32c(crc_stored, buf, 4);
1881 :
1882 1 : if (crc_computed != crc_stored) {
1883 1 : log_fatal("CRC mismatch in '%s'\n", path);
1884 1 : log_fatal("This content file is damaged! Please use an alternate copy.\n");
1885 1 : exit(EXIT_FAILURE);
1886 : } else {
1887 0 : log_fatal("The CRC of the file is correct!\n");
1888 : }
1889 0 : }
1890 :
1891 255 : static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
1892 : {
1893 : block_off_t blockmax;
1894 : unsigned count_file;
1895 : unsigned count_hardlink;
1896 : unsigned count_symlink;
1897 : unsigned count_dir;
1898 : int crc_checked;
1899 : char buffer[PATH_MAX];
1900 : int ret;
1901 : tommy_array disk_mapping;
1902 : uint32_t mapping_max;
1903 :
1904 255 : blockmax = 0;
1905 255 : count_file = 0;
1906 255 : count_hardlink = 0;
1907 255 : count_symlink = 0;
1908 255 : count_dir = 0;
1909 255 : crc_checked = 0;
1910 255 : mapping_max = 0;
1911 255 : tommy_array_init(&disk_mapping);
1912 :
1913 255 : ret = sread(f, buffer, 12);
1914 255 : if (ret < 0) {
1915 : /* LCOV_EXCL_START */
1916 : decoding_error(path, f);
1917 : log_fatal("Invalid header!\n");
1918 : os_abort();
1919 : /* LCOV_EXCL_STOP */
1920 : }
1921 :
1922 : /*
1923 : * File format versions:
1924 : * - SNAPCNT1/SnapRAID 4.0 First version.
1925 : * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
1926 : * The previous 'm' entry is now deprecated, but supported for importing.
1927 : * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
1928 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
1929 : * - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
1930 : * The previous 'P' entry is now deprecated, but supported for importing.
1931 : */
1932 255 : if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
1933 255 : && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
1934 255 : && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
1935 : ) {
1936 : /* LCOV_EXCL_START */
1937 : if (memcmp(buffer, "SNAPCNT", 7) != 0) {
1938 : decoding_error(path, f);
1939 : log_fatal("Invalid header!\n");
1940 : os_abort();
1941 : } else {
1942 : log_fatal("The content file '%s' was generated with a newer version of SnapRAID!\n", path);
1943 : exit(EXIT_FAILURE);
1944 : }
1945 : /* LCOV_EXCL_STOP */
1946 : }
1947 :
1948 3741541 : while (1) {
1949 : int c;
1950 :
1951 : /* read the command */
1952 3741796 : c = sgetc(f);
1953 3741796 : if (c == EOF) {
1954 254 : break;
1955 : }
1956 :
1957 3741542 : if (c == 'f') {
1958 : /* file */
1959 : char sub[PATH_MAX];
1960 : uint64_t v_size;
1961 : uint64_t v_mtime_sec;
1962 : uint32_t v_mtime_nsec;
1963 : uint64_t v_inode;
1964 : uint32_t v_idx;
1965 : struct snapraid_file* file;
1966 : struct snapraid_disk* disk;
1967 : uint32_t mapping;
1968 :
1969 3617706 : ret = sgetb32(f, &mapping);
1970 3617706 : if (ret < 0 || mapping >= mapping_max) {
1971 : /* LCOV_EXCL_START */
1972 : decoding_error(path, f);
1973 : log_fatal("Internal inconsistency: File mapping index out of range\n");
1974 : os_abort();
1975 : /* LCOV_EXCL_STOP */
1976 : }
1977 3617706 : disk = tommy_array_get(&disk_mapping, mapping);
1978 :
1979 3617706 : ret = sgetb64(f, &v_size);
1980 3617706 : if (ret < 0) {
1981 : /* LCOV_EXCL_START */
1982 : decoding_error(path, f);
1983 : os_abort();
1984 : /* LCOV_EXCL_STOP */
1985 : }
1986 :
1987 3617706 : if (state->block_size == 0) {
1988 : /* LCOV_EXCL_START */
1989 : decoding_error(path, f);
1990 : log_fatal("Internal inconsistency: Zero blocksize\n");
1991 : exit(EXIT_FAILURE);
1992 : /* LCOV_EXCL_STOP */
1993 : }
1994 :
1995 : /* check for impossible file size to avoid to crash for a too big allocation */
1996 3617706 : if (v_size / state->block_size > blockmax) {
1997 : /* LCOV_EXCL_START */
1998 : decoding_error(path, f);
1999 : log_fatal("Internal inconsistency: File size too big!\n");
2000 : os_abort();
2001 : /* LCOV_EXCL_STOP */
2002 : }
2003 :
2004 3617706 : ret = sgetb64(f, &v_mtime_sec);
2005 3617706 : if (ret < 0) {
2006 : /* LCOV_EXCL_START */
2007 : decoding_error(path, f);
2008 : os_abort();
2009 : /* LCOV_EXCL_STOP */
2010 : }
2011 :
2012 3617706 : ret = sgetb32(f, &v_mtime_nsec);
2013 3617706 : if (ret < 0) {
2014 : /* LCOV_EXCL_START */
2015 : decoding_error(path, f);
2016 : os_abort();
2017 : /* LCOV_EXCL_STOP */
2018 : }
2019 :
2020 : /* STAT_NSEC_INVALID is encoded as 0 */
2021 3617706 : if (v_mtime_nsec == 0)
2022 0 : v_mtime_nsec = STAT_NSEC_INVALID;
2023 : else
2024 3617706 : --v_mtime_nsec;
2025 :
2026 3617706 : ret = sgetb64(f, &v_inode);
2027 3617706 : if (ret < 0) {
2028 : /* LCOV_EXCL_START */
2029 : decoding_error(path, f);
2030 : os_abort();
2031 : /* LCOV_EXCL_STOP */
2032 : }
2033 :
2034 3617706 : ret = sgetbs(f, sub, sizeof(sub));
2035 3617706 : if (ret < 0) {
2036 : /* LCOV_EXCL_START */
2037 : decoding_error(path, f);
2038 : os_abort();
2039 : /* LCOV_EXCL_STOP */
2040 : }
2041 3617706 : if (!*sub) {
2042 : /* LCOV_EXCL_START */
2043 : decoding_error(path, f);
2044 : log_fatal("Internal inconsistency: Null file!\n");
2045 : os_abort();
2046 : /* LCOV_EXCL_STOP */
2047 : }
2048 :
2049 : /* allocate the file */
2050 3617706 : file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
2051 :
2052 : /* insert the file in the file containers */
2053 3617706 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
2054 3617706 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
2055 3617706 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
2056 3617706 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
2057 :
2058 : /* read all the blocks */
2059 3617706 : v_idx = 0;
2060 7256068 : while (v_idx < file->blockmax) {
2061 : block_off_t v_pos;
2062 : uint32_t v_count;
2063 :
2064 : /* get the "subcommand */
2065 3638362 : c = sgetc(f);
2066 :
2067 3638362 : ret = sgetb32(f, &v_pos);
2068 3638362 : if (ret < 0) {
2069 : /* LCOV_EXCL_START */
2070 : decoding_error(path, f);
2071 : os_abort();
2072 : /* LCOV_EXCL_STOP */
2073 : }
2074 :
2075 3638362 : ret = sgetb32(f, &v_count);
2076 3638362 : if (ret < 0) {
2077 : /* LCOV_EXCL_START */
2078 : decoding_error(path, f);
2079 : os_abort();
2080 : /* LCOV_EXCL_STOP */
2081 : }
2082 :
2083 3638362 : if (v_idx + v_count > file->blockmax) {
2084 : /* LCOV_EXCL_START */
2085 : decoding_error(path, f);
2086 : log_fatal("Internal inconsistency: Block number out of range\n");
2087 : os_abort();
2088 : /* LCOV_EXCL_STOP */
2089 : }
2090 :
2091 3638362 : if (v_pos + v_count > blockmax) {
2092 : /* LCOV_EXCL_START */
2093 : decoding_error(path, f);
2094 : log_fatal("Internal inconsistency: Block size %u/%u!\n", blockmax, v_pos + v_count);
2095 : os_abort();
2096 : /* LCOV_EXCL_START */
2097 : }
2098 :
2099 : /* fill the blocks in the run */
2100 : while (v_count) {
2101 : struct snapraid_block* block = fs_file2block_get(file, v_idx);
2102 :
2103 : switch (c) {
2104 : case 'b' :
2105 : block_state_set(block, BLOCK_STATE_BLK);
2106 : break;
2107 : case 'n' :
2108 : /* deprecated NEW blocks are converted to CHG ones */
2109 : block_state_set(block, BLOCK_STATE_CHG);
2110 : break;
2111 : case 'g' :
2112 : block_state_set(block, BLOCK_STATE_CHG);
2113 : break;
2114 : case 'p' :
2115 : block_state_set(block, BLOCK_STATE_REP);
2116 : break;
2117 : default :
2118 : /* LCOV_EXCL_START */
2119 : decoding_error(path, f);
2120 : log_fatal("Invalid block type!\n");
2121 : os_abort();
2122 : /* LCOV_EXCL_STOP */
2123 : }
2124 :
2125 : /* read the hash only for 'blk/chg/rep', and not for 'new' */
2126 8910861 : if (c != 'n') {
2127 8910861 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2128 8910861 : if (ret < 0) {
2129 : /* LCOV_EXCL_START */
2130 : decoding_error(path, f);
2131 : os_abort();
2132 : /* LCOV_EXCL_STOP */
2133 : }
2134 : } else {
2135 : /* set the ZERO hash for deprecated NEW blocks */
2136 0 : hash_zero_set(block->hash);
2137 : }
2138 :
2139 : /* if the block contains a hash of past data */
2140 : /* and we are clearing such indeterminate hashes */
2141 8910861 : if (state->clear_past_hash
2142 2542571 : && block_has_past_hash(block)
2143 : ) {
2144 : /* set the hash value to INVALID */
2145 28006 : hash_invalid_set(block->hash);
2146 : }
2147 :
2148 : /* if we are disabling the copy optimization */
2149 : /* we want also to clear any already previously stored information */
2150 : /* in other sync commands */
2151 : /* note that this is required only in sync, and we detect */
2152 : /* this using the clear_past_hash flag */
2153 8910861 : if (state->clear_past_hash
2154 2542571 : && state->opt.force_nocopy
2155 14195 : && block_state_get(block) == BLOCK_STATE_REP
2156 : ) {
2157 : /* set the hash value to INVALID */
2158 1 : hash_invalid_set(block->hash);
2159 : /* convert from REP to CHG block */
2160 1 : block_state_set(block, BLOCK_STATE_CHG);
2161 : }
2162 :
2163 : /* if we want a full reallocation, marks block as invalid parity */
2164 : /* note that we do this after the force_nocopy option */
2165 : /* to avoid to mixup the two things */
2166 8910861 : if (state->opt.force_realloc
2167 27826 : && block_state_get(block) == BLOCK_STATE_BLK) {
2168 : /* convert from BLK to REP */
2169 27826 : block_state_set(block, BLOCK_STATE_REP);
2170 : }
2171 :
2172 : /* set the parity association */
2173 8910861 : fs_allocate(disk, v_pos, file, v_idx);
2174 :
2175 : /* go to the next block */
2176 8910861 : ++v_idx;
2177 8910861 : ++v_pos;
2178 8910861 : --v_count;
2179 : }
2180 : }
2181 :
2182 : /* stat */
2183 3617706 : ++count_file;
2184 123836 : } else if (c == 'i') {
2185 : /* "inf" command */
2186 : snapraid_info info;
2187 : uint32_t v_pos;
2188 : uint64_t v_oldest;
2189 :
2190 254 : ret = sgetb64(f, &v_oldest);
2191 254 : if (ret < 0) {
2192 : /* LCOV_EXCL_START */
2193 : decoding_error(path, f);
2194 : os_abort();
2195 : /* LCOV_EXCL_STOP */
2196 : }
2197 :
2198 254 : v_pos = 0;
2199 42070 : while (v_pos < blockmax) {
2200 : int bad;
2201 : int rehash;
2202 : int justsynced;
2203 : uint64_t t64;
2204 : uint32_t flag;
2205 : uint32_t v_count;
2206 :
2207 41816 : ret = sgetb32(f, &v_count);
2208 41816 : if (ret < 0) {
2209 : /* LCOV_EXCL_START */
2210 : decoding_error(path, f);
2211 : os_abort();
2212 : /* LCOV_EXCL_STOP */
2213 : }
2214 :
2215 41816 : if (v_pos + v_count > blockmax) {
2216 : /* LCOV_EXCL_START */
2217 : decoding_error(path, f);
2218 : log_fatal("Internal inconsistency: Info size %u/%u!\n", blockmax, v_pos + v_count);
2219 : os_abort();
2220 : /* LCOV_EXCL_STOP */
2221 : }
2222 :
2223 41816 : ret = sgetb32(f, &flag);
2224 41816 : if (ret < 0) {
2225 : /* LCOV_EXCL_START */
2226 : decoding_error(path, f);
2227 : os_abort();
2228 : /* LCOV_EXCL_STOP */
2229 : }
2230 :
2231 : /* if there is an info */
2232 41816 : if ((flag & 1) != 0) {
2233 : /* read the time */
2234 41801 : ret = sgetb64(f, &t64);
2235 41801 : if (ret < 0) {
2236 : /* LCOV_EXCL_START */
2237 : decoding_error(path, f);
2238 : os_abort();
2239 : /* LCOV_EXCL_STOP */
2240 : }
2241 :
2242 : /* analyze the flags */
2243 41801 : bad = (flag & 2) != 0;
2244 41801 : rehash = (flag & 4) != 0;
2245 41801 : justsynced = (flag & 8) != 0;
2246 :
2247 41801 : if (rehash && state->prevhash == HASH_UNDEFINED) {
2248 : /* LCOV_EXCL_START */
2249 : decoding_error(path, f);
2250 : log_fatal("Internal inconsistency: Missing previous checksum!\n");
2251 : os_abort();
2252 : /* LCOV_EXCL_STOP */
2253 : }
2254 :
2255 41801 : info = info_make(t64 + v_oldest, bad, rehash, justsynced);
2256 : } else {
2257 15 : info = 0;
2258 : }
2259 :
2260 1592155 : while (v_count) {
2261 : /* insert the info in the array */
2262 1550339 : info_set(&state->infoarr, v_pos, info);
2263 :
2264 : /* ensure that an info is present only for used positions */
2265 1550339 : if (fs_info_is_required(state, v_pos)) {
2266 1491574 : if (!info) {
2267 : /* LCOV_EXCL_START */
2268 : decoding_error(path, f);
2269 : log_fatal("Internal inconsistency: Missing info!\n");
2270 : os_abort();
2271 : /* LCOV_EXCL_STOP */
2272 : }
2273 : } else {
2274 : /* extra info are accepted for backward compatibility */
2275 : /* they are discarded at the first write */
2276 : }
2277 :
2278 : /* go to next block */
2279 1550339 : ++v_pos;
2280 1550339 : --v_count;
2281 : }
2282 : }
2283 123582 : } else if (c == 'h') {
2284 : /* hole */
2285 : uint32_t v_pos;
2286 : struct snapraid_disk* disk;
2287 : uint32_t mapping;
2288 :
2289 1403 : ret = sgetb32(f, &mapping);
2290 1403 : if (ret < 0 || mapping >= mapping_max) {
2291 : /* LCOV_EXCL_START */
2292 : decoding_error(path, f);
2293 : log_fatal("Internal inconsistency: Hole mapping index out of range\n");
2294 : os_abort();
2295 : /* LCOV_EXCL_STOP */
2296 : }
2297 1403 : disk = tommy_array_get(&disk_mapping, mapping);
2298 :
2299 1403 : v_pos = 0;
2300 2872 : while (v_pos < blockmax) {
2301 : uint32_t v_idx;
2302 : uint32_t v_count;
2303 : struct snapraid_file* deleted;
2304 :
2305 1469 : ret = sgetb32(f, &v_count);
2306 1469 : if (ret < 0) {
2307 : /* LCOV_EXCL_START */
2308 : decoding_error(path, f);
2309 : os_abort();
2310 : /* LCOV_EXCL_STOP */
2311 : }
2312 :
2313 1469 : if (v_pos + v_count > blockmax) {
2314 : /* LCOV_EXCL_START */
2315 : decoding_error(path, f);
2316 : log_fatal("Internal inconsistency: Hole size %u/%u!\n", blockmax, v_pos + v_count);
2317 : os_abort();
2318 : /* LCOV_EXCL_STOP */
2319 : }
2320 :
2321 : /* get the sub-command */
2322 1469 : c = sgetc(f);
2323 :
2324 1469 : switch (c) {
2325 35 : case 'o' :
2326 : /* if it's a run of deleted blocks */
2327 :
2328 : /* allocate a fake deleted file */
2329 35 : deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
2330 :
2331 : /* mark the file as deleted */
2332 35 : file_flag_set(deleted, FILE_IS_DELETED);
2333 :
2334 : /* insert it in the list of deleted files */
2335 35 : tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
2336 :
2337 : /* process all blocks */
2338 35 : v_idx = 0;
2339 27921 : while (v_count) {
2340 27886 : struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
2341 :
2342 : /* set the block as deleted */
2343 27886 : block_state_set(block, BLOCK_STATE_DELETED);
2344 :
2345 : /* read the hash */
2346 27886 : ret = sread(f, block->hash, BLOCK_HASH_SIZE);
2347 27886 : if (ret < 0) {
2348 : /* LCOV_EXCL_START */
2349 : decoding_error(path, f);
2350 : os_abort();
2351 : /* LCOV_EXCL_STOP */
2352 : }
2353 :
2354 : /* if we are clearing indeterminate hashes */
2355 27886 : if (state->clear_past_hash) {
2356 : /* set the hash value to INVALID */
2357 9432 : hash_invalid_set(block->hash);
2358 : }
2359 :
2360 : /* insert the block in the block array */
2361 27886 : fs_allocate(disk, v_pos, deleted, v_idx);
2362 :
2363 : /* go to next block */
2364 27886 : ++v_pos;
2365 27886 : ++v_idx;
2366 27886 : --v_count;
2367 : }
2368 35 : break;
2369 1434 : case 'O' :
2370 : /* go to the next run */
2371 1434 : v_pos += v_count;
2372 1434 : break;
2373 0 : default :
2374 : /* LCOV_EXCL_START */
2375 : decoding_error(path, f);
2376 : log_fatal("Invalid hole type!\n");
2377 : os_abort();
2378 : /* LCOV_EXCL_STOP */
2379 : }
2380 : }
2381 122179 : } else if (c == 's') {
2382 : /* symlink */
2383 : char sub[PATH_MAX];
2384 : char linkto[PATH_MAX];
2385 : struct snapraid_link* slink;
2386 : struct snapraid_disk* disk;
2387 : uint32_t mapping;
2388 :
2389 116679 : ret = sgetb32(f, &mapping);
2390 116679 : if (ret < 0 || mapping >= mapping_max) {
2391 : /* LCOV_EXCL_START */
2392 : decoding_error(path, f);
2393 : log_fatal("Internal inconsistency: Symlink mapping index out of range\n");
2394 : os_abort();
2395 : /* LCOV_EXCL_STOP */
2396 : }
2397 116679 : disk = tommy_array_get(&disk_mapping, mapping);
2398 :
2399 116679 : ret = sgetbs(f, sub, sizeof(sub));
2400 116679 : if (ret < 0) {
2401 : /* LCOV_EXCL_START */
2402 : decoding_error(path, f);
2403 : os_abort();
2404 : /* LCOV_EXCL_STOP */
2405 : }
2406 :
2407 116679 : if (!*sub) {
2408 : /* LCOV_EXCL_START */
2409 : decoding_error(path, f);
2410 : log_fatal("Internal inconsistency: Null symlink!\n");
2411 : os_abort();
2412 : /* LCOV_EXCL_STOP */
2413 : }
2414 :
2415 116679 : ret = sgetbs(f, linkto, sizeof(linkto));
2416 116679 : if (ret < 0) {
2417 : /* LCOV_EXCL_START */
2418 : decoding_error(path, f);
2419 : os_abort();
2420 : /* LCOV_EXCL_STOP */
2421 : }
2422 :
2423 : /* allocate the link as symbolic link */
2424 116679 : slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
2425 :
2426 : /* insert the link in the link containers */
2427 116679 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2428 116679 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2429 :
2430 : /* stat */
2431 116679 : ++count_symlink;
2432 5500 : } else if (c == 'a') {
2433 : /* hardlink */
2434 : char sub[PATH_MAX];
2435 : char linkto[PATH_MAX];
2436 : struct snapraid_link* slink;
2437 : struct snapraid_disk* disk;
2438 : uint32_t mapping;
2439 :
2440 597 : ret = sgetb32(f, &mapping);
2441 597 : if (ret < 0 || mapping >= mapping_max) {
2442 : /* LCOV_EXCL_START */
2443 : decoding_error(path, f);
2444 : log_fatal("Internal inconsistency: Hardlink mapping index out of range!\n");
2445 : os_abort();
2446 : /* LCOV_EXCL_STOP */
2447 : }
2448 597 : disk = tommy_array_get(&disk_mapping, mapping);
2449 :
2450 597 : ret = sgetbs(f, sub, sizeof(sub));
2451 597 : if (ret < 0) {
2452 : /* LCOV_EXCL_START */
2453 : decoding_error(path, f);
2454 : os_abort();
2455 : /* LCOV_EXCL_STOP */
2456 : }
2457 :
2458 597 : if (!*sub) {
2459 : /* LCOV_EXCL_START */
2460 : decoding_error(path, f);
2461 : log_fatal("Internal inconsistency: Null hardlink!\n");
2462 : os_abort();
2463 : /* LCOV_EXCL_STOP */
2464 : }
2465 :
2466 597 : ret = sgetbs(f, linkto, sizeof(linkto));
2467 597 : if (ret < 0) {
2468 : /* LCOV_EXCL_START */
2469 : decoding_error(path, f);
2470 : os_abort();
2471 : /* LCOV_EXCL_STOP */
2472 : }
2473 :
2474 597 : if (!*linkto) {
2475 : /* LCOV_EXCL_START */
2476 : decoding_error(path, f);
2477 : log_fatal("Internal inconsistency: Empty hardlink '%s'!\n", sub);
2478 : os_abort();
2479 : /* LCOV_EXCL_STOP */
2480 : }
2481 :
2482 : /* allocate the link as hard link */
2483 597 : slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
2484 :
2485 : /* insert the link in the link containers */
2486 597 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
2487 597 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
2488 :
2489 : /* stat */
2490 597 : ++count_hardlink;
2491 4903 : } else if (c == 'r') {
2492 : /* dir */
2493 : char sub[PATH_MAX];
2494 : struct snapraid_dir* dir;
2495 : struct snapraid_disk* disk;
2496 : uint32_t mapping;
2497 :
2498 790 : ret = sgetb32(f, &mapping);
2499 790 : if (ret < 0 || mapping >= mapping_max) {
2500 : /* LCOV_EXCL_START */
2501 : decoding_error(path, f);
2502 : log_fatal("Internal inconsistency: Dir mapping index ouf of range!\n");
2503 : os_abort();
2504 : /* LCOV_EXCL_STOP */
2505 : }
2506 790 : disk = tommy_array_get(&disk_mapping, mapping);
2507 :
2508 790 : ret = sgetbs(f, sub, sizeof(sub));
2509 790 : if (ret < 0) {
2510 : /* LCOV_EXCL_START */
2511 : decoding_error(path, f);
2512 : os_abort();
2513 : /* LCOV_EXCL_STOP */
2514 : }
2515 :
2516 790 : if (!*sub) {
2517 : /* LCOV_EXCL_START */
2518 : decoding_error(path, f);
2519 : log_fatal("Internal inconsistency: Null dir!\n");
2520 : os_abort();
2521 : /* LCOV_EXCL_STOP */
2522 : }
2523 :
2524 : /* allocate the dir */
2525 790 : dir = dir_alloc(sub);
2526 :
2527 : /* insert the dir in the dir containers */
2528 790 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
2529 790 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
2530 :
2531 : /* stat */
2532 790 : ++count_dir;
2533 4113 : } else if (c == 'c') {
2534 : /* get the subcommand */
2535 255 : c = sgetc(f);
2536 :
2537 255 : switch (c) {
2538 140 : case 'u' :
2539 140 : state->hash = HASH_MURMUR3;
2540 140 : break;
2541 115 : case 'k' :
2542 115 : state->hash = HASH_SPOOKY2;
2543 115 : break;
2544 0 : case 'm' :
2545 0 : state->hash = HASH_METRO;
2546 0 : break;
2547 0 : default :
2548 : /* LCOV_EXCL_START */
2549 : decoding_error(path, f);
2550 : log_fatal("Invalid checksum!\n");
2551 : os_abort();
2552 : /* LCOV_EXCL_STOP */
2553 : }
2554 :
2555 : /* read the seed */
2556 255 : ret = sread(f, state->hashseed, HASH_MAX);
2557 255 : if (ret < 0) {
2558 : /* LCOV_EXCL_START */
2559 : decoding_error(path, f);
2560 : os_abort();
2561 : /* LCOV_EXCL_STOP */
2562 : }
2563 3858 : } else if (c == 'C') {
2564 : /* get the sub-command */
2565 5 : c = sgetc(f);
2566 :
2567 5 : switch (c) {
2568 4 : case 'u' :
2569 4 : state->prevhash = HASH_MURMUR3;
2570 4 : break;
2571 0 : case 'k' :
2572 0 : state->prevhash = HASH_SPOOKY2;
2573 0 : break;
2574 0 : case 'm' :
2575 0 : state->prevhash = HASH_METRO;
2576 0 : break;
2577 1 : default :
2578 : /* LCOV_EXCL_START */
2579 : decoding_error(path, f);
2580 : log_fatal("Invalid checksum!\n");
2581 : os_abort();
2582 : /* LCOV_EXCL_STOP */
2583 : }
2584 :
2585 : /* read the seed */
2586 4 : ret = sread(f, state->prevhashseed, HASH_MAX);
2587 4 : if (ret < 0) {
2588 : /* LCOV_EXCL_START */
2589 : decoding_error(path, f);
2590 : os_abort();
2591 : /* LCOV_EXCL_STOP */
2592 : }
2593 3853 : } else if (c == 'z') {
2594 : uint32_t block_size;
2595 :
2596 255 : ret = sgetb32(f, &block_size);
2597 255 : if (ret < 0) {
2598 : /* LCOV_EXCL_START */
2599 : decoding_error(path, f);
2600 : os_abort();
2601 : /* LCOV_EXCL_STOP */
2602 : }
2603 :
2604 255 : if (block_size == 0) {
2605 : /* LCOV_EXCL_START */
2606 : decoding_error(path, f);
2607 : log_fatal("Zero 'blocksize' specification in the content file!\n");
2608 : exit(EXIT_FAILURE);
2609 : /* LCOV_EXCL_STOP */
2610 : }
2611 :
2612 : /* without configuration, auto assign the block size */
2613 255 : if (state->no_conf) {
2614 1 : state->block_size = block_size;
2615 : }
2616 :
2617 255 : if (block_size != state->block_size) {
2618 : /* LCOV_EXCL_START */
2619 : decoding_error(path, f);
2620 : log_fatal("Mismatching 'blocksize' specification in the content file!\n");
2621 : log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
2622 : exit(EXIT_FAILURE);
2623 : /* LCOV_EXCL_STOP */
2624 : }
2625 3598 : } else if (c == 'y') {
2626 : uint32_t hash_size;
2627 :
2628 255 : ret = sgetb32(f, &hash_size);
2629 255 : if (ret < 0) {
2630 : /* LCOV_EXCL_START */
2631 : decoding_error(path, f);
2632 : os_abort();
2633 : /* LCOV_EXCL_STOP */
2634 : }
2635 :
2636 255 : if (hash_size < 2 || hash_size > HASH_MAX) {
2637 : /* LCOV_EXCL_START */
2638 : decoding_error(path, f);
2639 : log_fatal("Invalid 'hashsize' specification in the content file!\n");
2640 : exit(EXIT_FAILURE);
2641 : /* LCOV_EXCL_STOP */
2642 : }
2643 :
2644 : /* without configuration, auto assign the block size */
2645 255 : if (state->no_conf) {
2646 1 : BLOCK_HASH_SIZE = hash_size;
2647 : }
2648 :
2649 255 : if ((int)hash_size != BLOCK_HASH_SIZE) {
2650 : /* LCOV_EXCL_START */
2651 : decoding_error(path, f);
2652 : log_fatal("Mismatching 'hashsize' specification in the content file!\n");
2653 : log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
2654 : exit(EXIT_FAILURE);
2655 : /* LCOV_EXCL_STOP */
2656 : }
2657 3343 : } else if (c == 'x') {
2658 255 : ret = sgetb32(f, &blockmax);
2659 255 : if (ret < 0) {
2660 : /* LCOV_EXCL_START */
2661 : decoding_error(path, f);
2662 : os_abort();
2663 : /* LCOV_EXCL_STOP */
2664 : }
2665 3088 : } else if (c == 'm' || c == 'M') {
2666 : struct snapraid_map* map;
2667 : char uuid[UUID_MAX];
2668 : uint32_t v_pos;
2669 : uint32_t v_total_blocks;
2670 : uint32_t v_free_blocks;
2671 : struct snapraid_disk* disk;
2672 :
2673 1409 : ret = sgetbs(f, buffer, sizeof(buffer));
2674 1409 : if (ret < 0) {
2675 : /* LCOV_EXCL_START */
2676 : decoding_error(path, f);
2677 : os_abort();
2678 : /* LCOV_EXCL_STOP */
2679 : }
2680 :
2681 1409 : ret = sgetb32(f, &v_pos);
2682 1409 : if (ret < 0) {
2683 : /* LCOV_EXCL_START */
2684 : decoding_error(path, f);
2685 : os_abort();
2686 : /* LCOV_EXCL_STOP */
2687 : }
2688 :
2689 : /* from SnapRAID 7.0 the 'M' command includes the free space */
2690 1409 : if (c == 'M') {
2691 1409 : ret = sgetb32(f, &v_total_blocks);
2692 1409 : if (ret < 0) {
2693 : /* LCOV_EXCL_START */
2694 : decoding_error(path, f);
2695 : os_abort();
2696 : /* LCOV_EXCL_STOP */
2697 : }
2698 :
2699 1409 : ret = sgetb32(f, &v_free_blocks);
2700 1409 : if (ret < 0) {
2701 : /* LCOV_EXCL_START */
2702 : decoding_error(path, f);
2703 : os_abort();
2704 : /* LCOV_EXCL_STOP */
2705 : }
2706 : } else {
2707 0 : v_total_blocks = 0;
2708 0 : v_free_blocks = 0;
2709 : }
2710 :
2711 : /* read the uuid */
2712 1409 : ret = sgetbs(f, uuid, sizeof(uuid));
2713 1409 : if (ret < 0) {
2714 : /* LCOV_EXCL_START */
2715 : decoding_error(path, f);
2716 : os_abort();
2717 : /* LCOV_EXCL_STOP */
2718 : }
2719 :
2720 : /* find the disk */
2721 1409 : disk = find_disk_by_name(state, buffer);
2722 1409 : if (!disk) {
2723 : /* search by UUID if renamed */
2724 2 : disk = find_disk_by_uuid(state, uuid);
2725 2 : if (disk) {
2726 2 : log_fatal("WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
2727 :
2728 : /* write the new state with the new name */
2729 2 : state->need_write = 1;
2730 : }
2731 : }
2732 1409 : if (!disk) {
2733 : /* LCOV_EXCL_START */
2734 : decoding_error(path, f);
2735 : log_fatal("Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
2736 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2737 : /* if it's a command without UUID, it cannot autorename using UUID */
2738 : if (state->opt.skip_disk_access)
2739 : log_fatal("If you have renamed it, run 'sync' to update the new name\n");
2740 : exit(EXIT_FAILURE);
2741 : /* LCOV_EXCL_STOP */
2742 : }
2743 :
2744 1409 : map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
2745 :
2746 1409 : tommy_list_insert_tail(&state->maplist, &map->node, map);
2747 :
2748 : /* insert in the mapping vector */
2749 1409 : tommy_array_grow(&disk_mapping, mapping_max + 1);
2750 1409 : tommy_array_set(&disk_mapping, mapping_max, disk);
2751 1409 : ++mapping_max;
2752 1679 : } else if (c == 'P') {
2753 : /* from SnapRAID 7.0 the 'P' command includes the free space */
2754 : /* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
2755 : char v_uuid[UUID_MAX];
2756 : uint32_t v_level;
2757 : uint32_t v_total_blocks;
2758 : uint32_t v_free_blocks;
2759 :
2760 0 : ret = sgetb32(f, &v_level);
2761 0 : if (ret < 0) {
2762 : /* LCOV_EXCL_START */
2763 : decoding_error(path, f);
2764 : os_abort();
2765 : /* LCOV_EXCL_STOP */
2766 : }
2767 :
2768 0 : ret = sgetb32(f, &v_total_blocks);
2769 0 : if (ret < 0) {
2770 : /* LCOV_EXCL_START */
2771 : decoding_error(path, f);
2772 : os_abort();
2773 : /* LCOV_EXCL_STOP */
2774 : }
2775 :
2776 0 : ret = sgetb32(f, &v_free_blocks);
2777 0 : if (ret < 0) {
2778 : /* LCOV_EXCL_START */
2779 : decoding_error(path, f);
2780 : os_abort();
2781 : /* LCOV_EXCL_STOP */
2782 : }
2783 :
2784 0 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2785 0 : if (ret < 0) {
2786 : /* LCOV_EXCL_START */
2787 : decoding_error(path, f);
2788 : os_abort();
2789 : /* LCOV_EXCL_STOP */
2790 : }
2791 :
2792 0 : if (v_level >= LEV_MAX) {
2793 : /* LCOV_EXCL_START */
2794 : decoding_error(path, f);
2795 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2796 : exit(EXIT_FAILURE);
2797 : /* LCOV_EXCL_STOP */
2798 : }
2799 :
2800 : /* auto configure if configuration is missing */
2801 0 : if (state->no_conf) {
2802 0 : if (v_level >= state->level)
2803 0 : state->level = v_level + 1;
2804 : }
2805 :
2806 : /* if we use this parity entry */
2807 0 : if (v_level < state->level) {
2808 : /* if the configuration has more splits, keep them */
2809 0 : if (state->parity[v_level].split_mac < 1)
2810 0 : state->parity[v_level].split_mac = 1;
2811 : /* set the parity info */
2812 0 : pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
2813 0 : state->parity[v_level].total_blocks = v_total_blocks;
2814 0 : state->parity[v_level].free_blocks = v_free_blocks;
2815 : }
2816 1679 : } else if (c == 'Q') {
2817 : /* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
2818 : uint32_t v_level;
2819 : uint32_t v_total_blocks;
2820 : uint32_t v_free_blocks;
2821 : uint32_t v_split_mac;
2822 : unsigned s;
2823 :
2824 1425 : ret = sgetb32(f, &v_level);
2825 1425 : if (ret < 0) {
2826 : /* LCOV_EXCL_START */
2827 : decoding_error(path, f);
2828 : os_abort();
2829 : /* LCOV_EXCL_STOP */
2830 : }
2831 :
2832 1425 : ret = sgetb32(f, &v_total_blocks);
2833 1425 : if (ret < 0) {
2834 : /* LCOV_EXCL_START */
2835 : decoding_error(path, f);
2836 : os_abort();
2837 : /* LCOV_EXCL_STOP */
2838 : }
2839 :
2840 1425 : ret = sgetb32(f, &v_free_blocks);
2841 1425 : if (ret < 0) {
2842 : /* LCOV_EXCL_START */
2843 : decoding_error(path, f);
2844 : os_abort();
2845 : /* LCOV_EXCL_STOP */
2846 : }
2847 :
2848 1425 : ret = sgetb32(f, &v_split_mac);
2849 1425 : if (ret < 0) {
2850 : /* LCOV_EXCL_START */
2851 : decoding_error(path, f);
2852 : os_abort();
2853 : /* LCOV_EXCL_STOP */
2854 : }
2855 :
2856 1425 : if (v_level >= LEV_MAX) {
2857 : /* LCOV_EXCL_START */
2858 : decoding_error(path, f);
2859 : log_fatal("Invalid parity level '%u' in the configuration file!\n", v_level);
2860 : exit(EXIT_FAILURE);
2861 : /* LCOV_EXCL_STOP */
2862 : }
2863 :
2864 : /* auto configure if configuration is missing */
2865 1425 : if (state->no_conf) {
2866 6 : if (v_level >= state->level)
2867 5 : state->level = v_level + 1;
2868 6 : if (state->parity[v_level].split_mac < v_split_mac)
2869 6 : state->parity[v_level].split_mac = v_split_mac;
2870 : }
2871 :
2872 : /* if we use this parity entry */
2873 1425 : if (v_level < state->level) {
2874 : /* set the parity info */
2875 1166 : state->parity[v_level].total_blocks = v_total_blocks;
2876 1166 : state->parity[v_level].free_blocks = v_free_blocks;
2877 : }
2878 :
2879 7125 : for (s = 0; s < v_split_mac; ++s) {
2880 : char v_path[PATH_MAX];
2881 : char v_uuid[UUID_MAX];
2882 : uint64_t v_size;
2883 :
2884 5700 : ret = sgetbs(f, v_path, sizeof(v_path));
2885 5700 : if (ret < 0) {
2886 : /* LCOV_EXCL_START */
2887 : decoding_error(path, f);
2888 : os_abort();
2889 : /* LCOV_EXCL_STOP */
2890 : }
2891 :
2892 5700 : ret = sgetbs(f, v_uuid, sizeof(v_uuid));
2893 5700 : if (ret < 0) {
2894 : /* LCOV_EXCL_START */
2895 : decoding_error(path, f);
2896 : os_abort();
2897 : /* LCOV_EXCL_STOP */
2898 : }
2899 :
2900 5700 : ret = sgetb64(f, &v_size);
2901 5700 : if (ret < 0) {
2902 : /* LCOV_EXCL_START */
2903 : decoding_error(path, f);
2904 : os_abort();
2905 : /* LCOV_EXCL_STOP */
2906 : }
2907 :
2908 : /* if we use this parity entry */
2909 5700 : if (v_level < state->level) {
2910 : /* if this split was removed from the configuration */
2911 4664 : if (s >= state->parity[v_level].split_mac) {
2912 : /* if the file is used, we really need it */
2913 0 : if (v_size != 0) {
2914 : /* LCOV_EXCL_START */
2915 : decoding_error(path, f);
2916 : log_fatal("Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
2917 : log_fatal("If you have removed it from the configuration file, please restore it\n");
2918 : exit(EXIT_FAILURE);
2919 : /* LCOV_EXCL_STOP */
2920 : }
2921 :
2922 : /* otherwise we can drop it */
2923 0 : log_fatal("WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
2924 : } else {
2925 : /* we copy the path only if without configuration file */
2926 4664 : if (state->no_conf)
2927 24 : pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
2928 :
2929 : /* set the split info */
2930 4664 : pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
2931 4664 : state->parity[v_level].split_map[s].size = v_size;
2932 :
2933 : /* log the info read from the content file */
2934 4664 : log_tag("content:%s:%u:%s:%s:%" PRIi64 "\n", lev_config_name(v_level), s,
2935 4664 : state->parity[v_level].split_map[s].path,
2936 4664 : state->parity[v_level].split_map[s].uuid,
2937 : state->parity[v_level].split_map[s].size);
2938 : }
2939 : }
2940 : }
2941 254 : } else if (c == 'N') {
2942 : uint32_t crc_stored;
2943 : uint32_t crc_computed;
2944 :
2945 : /* get the crc before reading it from the file */
2946 254 : crc_computed = scrc(f);
2947 :
2948 254 : ret = sgetble32(f, &crc_stored);
2949 254 : if (ret < 0) {
2950 : /* LCOV_EXCL_START */
2951 : /* here don't call decoding_error() because it's too late to get the crc */
2952 : log_fatal("Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
2953 : log_fatal("This content file is damaged! Use an alternate copy.\n");
2954 : exit(EXIT_FAILURE);
2955 : /* LCOV_EXCL_STOP */
2956 : }
2957 :
2958 254 : if (crc_stored != crc_computed) {
2959 : /* LCOV_EXCL_START */
2960 : /* here don't call decoding_error() because it's too late to get the crc */
2961 : log_fatal("CRC mismatch in '%s'\n", path);
2962 : log_fatal("The content file is damaged! Please use an alternate copy.\n");
2963 : exit(EXIT_FAILURE);
2964 : /* LCOV_EXCL_STOP */
2965 : }
2966 :
2967 254 : crc_checked = 1;
2968 : } else {
2969 : /* LCOV_EXCL_START */
2970 : decoding_error(path, f);
2971 : log_fatal("Invalid command '%c'!\n", (char)c);
2972 : os_abort();
2973 : /* LCOV_EXCL_STOP */
2974 : }
2975 : }
2976 :
2977 254 : tommy_array_done(&disk_mapping);
2978 :
2979 254 : if (serror(f)) {
2980 : /* LCOV_EXCL_START */
2981 : log_fatal("Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
2982 : exit(EXIT_FAILURE);
2983 : /* LCOV_EXCL_STOP */
2984 : }
2985 :
2986 254 : if (!crc_checked) {
2987 : /* LCOV_EXCL_START */
2988 : log_fatal("Reached the end of '%s' without finding the expected CRC\n", path);
2989 : log_fatal("This content file is truncated or damaged! Use an alternate copy.\n");
2990 : exit(EXIT_FAILURE);
2991 : /* LCOV_EXCL_STOP */
2992 : }
2993 :
2994 : /* check the file-system on all disks */
2995 254 : state_fscheck(state, "after read");
2996 :
2997 : /* check that the stored parity size matches the loaded state */
2998 254 : if (blockmax != parity_allocated_size(state)) {
2999 : /* LCOV_EXCL_START */
3000 : log_fatal("Internal inconsistency: Parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
3001 : if (state->opt.skip_content_check) {
3002 : log_fatal("Overriding.\n");
3003 : blockmax = parity_allocated_size(state);
3004 : } else {
3005 : exit(EXIT_FAILURE);
3006 : }
3007 : /* LCOV_EXCL_STOP */
3008 : }
3009 :
3010 254 : msg_verbose("%8u files\n", count_file);
3011 254 : msg_verbose("%8u hardlinks\n", count_hardlink);
3012 254 : msg_verbose("%8u symlinks\n", count_symlink);
3013 254 : msg_verbose("%8u empty dirs\n", count_dir);
3014 254 : }
3015 :
3016 : struct state_write_thread_context {
3017 : struct snapraid_state* state;
3018 : #if HAVE_MT_WRITE
3019 : thread_id_t thread;
3020 : #endif
3021 : /* input */
3022 : block_off_t blockmax;
3023 : time_t info_oldest;
3024 : time_t info_now;
3025 : int info_has_rehash;
3026 : STREAM* f;
3027 : /* output */
3028 : uint32_t crc;
3029 : unsigned count_file;
3030 : unsigned count_hardlink;
3031 : unsigned count_symlink;
3032 : unsigned count_dir;
3033 : };
3034 :
3035 146 : static void* state_write_thread(void* arg)
3036 : {
3037 146 : struct state_write_thread_context* context = arg;
3038 146 : struct snapraid_state* state = context->state;
3039 146 : block_off_t blockmax = context->blockmax;
3040 146 : time_t info_oldest = context->info_oldest;
3041 146 : time_t info_now = context->info_now;
3042 146 : int info_has_rehash = context->info_has_rehash;
3043 146 : STREAM* f = context->f;
3044 : uint32_t crc;
3045 : uint64_t t64;
3046 : unsigned count_file;
3047 : unsigned count_hardlink;
3048 : unsigned count_symlink;
3049 : unsigned count_dir;
3050 : tommy_node* i;
3051 : block_off_t idx;
3052 : block_off_t begin;
3053 : unsigned l, s;
3054 : int version;
3055 :
3056 146 : count_file = 0;
3057 146 : count_hardlink = 0;
3058 146 : count_symlink = 0;
3059 146 : count_dir = 0;
3060 :
3061 : /* check what version to use */
3062 146 : version = 2;
3063 977 : for (l = 0; l < state->level; ++l) {
3064 831 : if (state->parity[l].split_mac > 1)
3065 831 : version = 3;
3066 : }
3067 146 : if (BLOCK_HASH_SIZE != 16)
3068 0 : version = 3;
3069 :
3070 : /* write header */
3071 146 : if (version == 3)
3072 146 : swrite("SNAPCNT3\n\3\0\0", 12, f);
3073 : else
3074 0 : swrite("SNAPCNT2\n\3\0\0", 12, f);
3075 :
3076 : /* write block size and block max */
3077 146 : sputc('z', f);
3078 146 : sputb32(state->block_size, f);
3079 146 : sputc('x', f);
3080 146 : sputb32(blockmax, f);
3081 :
3082 : /* hash size */
3083 146 : if (version == 3) {
3084 146 : sputc('y', f);
3085 146 : sputb32(BLOCK_HASH_SIZE, f);
3086 : }
3087 :
3088 146 : if (serror(f)) {
3089 : /* LCOV_EXCL_START */
3090 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3091 : return context;
3092 : /* LCOV_EXCL_STOP */
3093 : }
3094 :
3095 146 : sputc('c', f);
3096 146 : if (state->hash == HASH_MURMUR3) {
3097 91 : sputc('u', f);
3098 55 : } else if (state->hash == HASH_SPOOKY2) {
3099 55 : sputc('k', f);
3100 0 : } else if (state->hash == HASH_METRO) {
3101 0 : sputc('m', f);
3102 : } else {
3103 : /* LCOV_EXCL_START */
3104 : log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
3105 : return context;
3106 : /* LCOV_EXCL_STOP */
3107 : }
3108 146 : swrite(state->hashseed, HASH_MAX, f);
3109 146 : if (serror(f)) {
3110 : /* LCOV_EXCL_START */
3111 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3112 : return context;
3113 : /* LCOV_EXCL_STOP */
3114 : }
3115 :
3116 : /* previous hash only present */
3117 146 : if (state->prevhash != HASH_UNDEFINED) {
3118 : /* if at least one rehash tag found, we have to save the previous hash */
3119 3 : if (info_has_rehash) {
3120 2 : sputc('C', f);
3121 2 : if (state->prevhash == HASH_MURMUR3) {
3122 2 : sputc('u', f);
3123 0 : } else if (state->prevhash == HASH_SPOOKY2) {
3124 0 : sputc('k', f);
3125 0 : } else if (state->prevhash == HASH_METRO) {
3126 0 : sputc('m', f);
3127 : } else {
3128 : /* LCOV_EXCL_START */
3129 : log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
3130 : return context;
3131 : /* LCOV_EXCL_STOP */
3132 : }
3133 2 : swrite(state->prevhashseed, HASH_MAX, f);
3134 2 : if (serror(f)) {
3135 : /* LCOV_EXCL_START */
3136 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3137 : return context;
3138 : /* LCOV_EXCL_STOP */
3139 : }
3140 : }
3141 : }
3142 :
3143 : /* for each map */
3144 1022 : for (i = state->maplist; i != 0; i = i->next) {
3145 876 : struct snapraid_map* map = i->data;
3146 : struct snapraid_disk* disk;
3147 :
3148 : /* find the disk for this mapping */
3149 876 : disk = find_disk_by_name(state, map->name);
3150 876 : if (!disk) {
3151 : /* LCOV_EXCL_START */
3152 : log_fatal("Internal inconsistency: Unmapped disk '%s'\n", map->name);
3153 : return context;
3154 : /* LCOV_EXCL_STOP */
3155 : }
3156 :
3157 : /* save the mapping only if disk is mapped */
3158 876 : if (disk->mapping_idx != -1) {
3159 784 : sputc('M', f);
3160 784 : sputbs(map->name, f);
3161 784 : sputb32(map->position, f);
3162 784 : sputb32(map->total_blocks, f);
3163 784 : sputb32(map->free_blocks, f);
3164 784 : sputbs(map->uuid, f);
3165 784 : if (serror(f)) {
3166 : /* LCOV_EXCL_START */
3167 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3168 : return context;
3169 : /* LCOV_EXCL_STOP */
3170 : }
3171 : }
3172 : }
3173 :
3174 : /* for each parity */
3175 977 : for (l = 0; l < state->level; ++l) {
3176 831 : if (version == 3) {
3177 831 : sputc('Q', f);
3178 831 : sputb32(l, f);
3179 831 : sputb32(state->parity[l].total_blocks, f);
3180 831 : sputb32(state->parity[l].free_blocks, f);
3181 831 : sputb32(state->parity[l].split_mac, f);
3182 4155 : for (s = 0; s < state->parity[l].split_mac; ++s) {
3183 3324 : sputbs(state->parity[l].split_map[s].path, f);
3184 3324 : sputbs(state->parity[l].split_map[s].uuid, f);
3185 3324 : sputb64(state->parity[l].split_map[s].size, f);
3186 : }
3187 : } else {
3188 0 : sputc('P', f);
3189 0 : sputb32(l, f);
3190 0 : sputb32(state->parity[l].total_blocks, f);
3191 0 : sputb32(state->parity[l].free_blocks, f);
3192 0 : sputbs(state->parity[l].split_map[0].uuid, f);
3193 : }
3194 831 : if (serror(f)) {
3195 : /* LCOV_EXCL_START */
3196 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3197 : return context;
3198 : /* LCOV_EXCL_STOP */
3199 : }
3200 : }
3201 :
3202 : /* for each disk */
3203 1022 : for (i = state->disklist; i != 0; i = i->next) {
3204 : tommy_node* j;
3205 876 : struct snapraid_disk* disk = i->data;
3206 :
3207 : /* if the disk is not mapped, skip it */
3208 876 : if (disk->mapping_idx < 0)
3209 92 : continue;
3210 :
3211 : /* for each file */
3212 1720845 : for (j = disk->filelist; j != 0; j = j->next) {
3213 1720061 : struct snapraid_file* file = j->data;
3214 : uint64_t size;
3215 : uint64_t mtime_sec;
3216 : int32_t mtime_nsec;
3217 : uint64_t inode;
3218 :
3219 1720061 : size = file->size;
3220 1720061 : mtime_sec = file->mtime_sec;
3221 1720061 : mtime_nsec = file->mtime_nsec;
3222 1720061 : inode = file->inode;
3223 :
3224 1720061 : sputc('f', f);
3225 1720061 : sputb32(disk->mapping_idx, f);
3226 1720061 : sputb64(size, f);
3227 1720061 : sputb64(mtime_sec, f);
3228 : /* encode STAT_NSEC_INVALID as 0 */
3229 1720061 : if (mtime_nsec == STAT_NSEC_INVALID)
3230 0 : sputb32(0, f);
3231 : else
3232 1720061 : sputb32(mtime_nsec + 1, f);
3233 1720061 : sputb64(inode, f);
3234 1720061 : sputbs(file->sub, f);
3235 1720061 : if (serror(f)) {
3236 : /* LCOV_EXCL_START */
3237 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3238 : return context;
3239 : /* LCOV_EXCL_STOP */
3240 : }
3241 :
3242 : /* for all the blocks of the file */
3243 1720061 : begin = 0;
3244 3450166 : while (begin < file->blockmax) {
3245 1730105 : unsigned v_state = block_state_get(fs_file2block_get(file, begin));
3246 1730105 : block_off_t v_pos = fs_file2par_get(disk, file, begin);
3247 : uint32_t v_count;
3248 :
3249 : block_off_t end;
3250 :
3251 : /* find the end of run of blocks */
3252 1730105 : end = begin + 1;
3253 4239641 : while (end < file->blockmax) {
3254 2521977 : if (v_state != block_state_get(fs_file2block_get(file, end)))
3255 12 : break;
3256 2521965 : if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
3257 12429 : break;
3258 2509536 : ++end;
3259 : }
3260 :
3261 1730105 : switch (v_state) {
3262 1629262 : case BLOCK_STATE_BLK :
3263 1629262 : sputc('b', f);
3264 1629262 : break;
3265 65926 : case BLOCK_STATE_CHG :
3266 65926 : sputc('g', f);
3267 65926 : break;
3268 34917 : case BLOCK_STATE_REP :
3269 34917 : sputc('p', f);
3270 34917 : break;
3271 0 : default :
3272 : /* LCOV_EXCL_START */
3273 : log_fatal("Internal inconsistency: State for block %u state %u\n", v_pos, v_state);
3274 : return context;
3275 : /* LCOV_EXCL_STOP */
3276 : }
3277 :
3278 1730105 : sputb32(v_pos, f);
3279 :
3280 1730105 : v_count = end - begin;
3281 1730105 : sputb32(v_count, f);
3282 :
3283 : /* write hashes */
3284 5969746 : for (idx = begin; idx < end; ++idx) {
3285 4239641 : struct snapraid_block* block = fs_file2block_get(file, idx);
3286 :
3287 4239641 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3288 : }
3289 :
3290 1730105 : if (serror(f)) {
3291 : /* LCOV_EXCL_START */
3292 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3293 : return context;
3294 : /* LCOV_EXCL_STOP */
3295 : }
3296 :
3297 : /* next begin position */
3298 1730105 : begin = end;
3299 : }
3300 :
3301 1720061 : ++count_file;
3302 : }
3303 :
3304 : /* for each link */
3305 57134 : for (j = disk->linklist; j != 0; j = j->next) {
3306 56350 : struct snapraid_link* slink = j->data;
3307 :
3308 56350 : switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
3309 309 : case FILE_IS_HARDLINK :
3310 309 : sputc('a', f);
3311 309 : ++count_hardlink;
3312 309 : break;
3313 56041 : case FILE_IS_SYMLINK :
3314 56041 : sputc('s', f);
3315 56041 : ++count_symlink;
3316 56041 : break;
3317 : }
3318 :
3319 56350 : sputb32(disk->mapping_idx, f);
3320 56350 : sputbs(slink->sub, f);
3321 56350 : sputbs(slink->linkto, f);
3322 56350 : if (serror(f)) {
3323 : /* LCOV_EXCL_START */
3324 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3325 : return context;
3326 : /* LCOV_EXCL_STOP */
3327 : }
3328 : }
3329 :
3330 : /* for each dir */
3331 1195 : for (j = disk->dirlist; j != 0; j = j->next) {
3332 411 : struct snapraid_dir* dir = j->data;
3333 :
3334 411 : sputc('r', f);
3335 411 : sputb32(disk->mapping_idx, f);
3336 411 : sputbs(dir->sub, f);
3337 411 : if (serror(f)) {
3338 : /* LCOV_EXCL_START */
3339 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3340 : return context;
3341 : /* LCOV_EXCL_STOP */
3342 : }
3343 :
3344 411 : ++count_dir;
3345 : }
3346 :
3347 : /* deleted blocks of the disk */
3348 784 : sputc('h', f);
3349 784 : sputb32(disk->mapping_idx, f);
3350 784 : if (serror(f)) {
3351 : /* LCOV_EXCL_START */
3352 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3353 : return context;
3354 : /* LCOV_EXCL_STOP */
3355 : }
3356 784 : begin = 0;
3357 1642 : while (begin < blockmax) {
3358 : int is_deleted;
3359 : block_off_t end;
3360 :
3361 858 : is_deleted = fs_is_block_deleted(disk, begin);
3362 :
3363 : /* find the end of run of blocks */
3364 858 : end = begin + 1;
3365 858 : while (end < blockmax
3366 4402569 : && is_deleted == fs_is_block_deleted(disk, end)
3367 : ) {
3368 4401711 : ++end;
3369 : }
3370 :
3371 858 : sputb32(end - begin, f);
3372 :
3373 858 : if (is_deleted) {
3374 : /* write the run of deleted blocks with hash */
3375 47 : sputc('o', f);
3376 :
3377 : /* write all the hash */
3378 15914 : while (begin < end) {
3379 15867 : struct snapraid_block* block = fs_par2block_get(disk, begin);
3380 :
3381 15867 : swrite(block->hash, BLOCK_HASH_SIZE, f);
3382 :
3383 15867 : ++begin;
3384 : }
3385 : } else {
3386 : /* write the run of blocks without hash */
3387 : /* they can be either used or empty blocks */
3388 811 : sputc('O', f);
3389 :
3390 : /* next begin position */
3391 811 : begin = end;
3392 : }
3393 :
3394 858 : if (serror(f)) {
3395 : /* LCOV_EXCL_START */
3396 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3397 : return context;
3398 : /* LCOV_EXCL_STOP */
3399 : }
3400 : }
3401 : }
3402 :
3403 : /* write the info for each block */
3404 146 : sputc('i', f);
3405 : /* ensure to write a 64 bit time */
3406 146 : t64 = info_oldest;
3407 146 : sputb64(t64, f);
3408 146 : begin = 0;
3409 14456 : while (begin < blockmax) {
3410 : snapraid_info info;
3411 : block_off_t end;
3412 : time_t t;
3413 : unsigned flag;
3414 :
3415 14310 : info = info_get(&state->infoarr, begin);
3416 :
3417 : /* find the end of run of blocks */
3418 14310 : end = begin + 1;
3419 14310 : while (end < blockmax
3420 734936 : && info == info_get(&state->infoarr, end)
3421 : ) {
3422 720626 : ++end;
3423 : }
3424 :
3425 14310 : sputb32(end - begin, f);
3426 :
3427 : /* if there is info */
3428 14310 : if (info) {
3429 : /* other flags */
3430 14287 : flag = 1; /* info is present */
3431 14287 : if (info_get_bad(info))
3432 1334 : flag |= 2;
3433 14287 : if (info_get_rehash(info))
3434 4967 : flag |= 4;
3435 14287 : if (info_get_justsynced(info))
3436 7599 : flag |= 8;
3437 14287 : sputb32(flag, f);
3438 :
3439 14287 : t = info_get_time(info);
3440 :
3441 : /* truncate any time that is in the future */
3442 14287 : if (t > info_now)
3443 0 : t = info_now;
3444 :
3445 : /* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
3446 14287 : if (t < info_oldest)
3447 0 : t = 0;
3448 : else
3449 14287 : t -= info_oldest;
3450 :
3451 : /* ensure to write a 64 bit time */
3452 14287 : t64 = t;
3453 14287 : sputb64(t64, f);
3454 : } else {
3455 : /* write a special 0 flag to mark missing info */
3456 23 : sputb32(0, f);
3457 : }
3458 :
3459 14310 : if (serror(f)) {
3460 : /* LCOV_EXCL_START */
3461 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3462 : return context;
3463 : /* LCOV_EXCL_STOP */
3464 : }
3465 :
3466 : /* next begin position */
3467 14310 : begin = end;
3468 : }
3469 :
3470 146 : sputc('N', f);
3471 :
3472 : /* flush data written to the disk */
3473 146 : if (sflush(f)) {
3474 : /* LCOV_EXCL_START */
3475 : log_fatal("Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
3476 : return context;
3477 : /* LCOV_EXCL_STOP */
3478 : }
3479 :
3480 : /* get the file crc */
3481 146 : crc = scrc(f);
3482 :
3483 : /* compare the crc of the data written to file */
3484 : /* with the one of the data written to the stream */
3485 146 : if (crc != scrc_stream(f)) {
3486 : /* LCOV_EXCL_START */
3487 : log_fatal("CRC mismatch while writing the content stream.\n");
3488 : log_fatal("DANGER! Your RAM memory is faulty! DO NOT PROCEED UNTIL FIXED!\n");
3489 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3490 : return context;
3491 : /* LCOV_EXCL_STOP */
3492 : }
3493 :
3494 146 : sputble32(crc, f);
3495 146 : if (serror(f)) {
3496 : /* LCOV_EXCL_START */
3497 : log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
3498 : return context;
3499 : /* LCOV_EXCL_STOP */
3500 : }
3501 :
3502 : /* set output variables */
3503 146 : context->crc = crc;
3504 146 : context->count_file = count_file;
3505 146 : context->count_hardlink = count_hardlink;
3506 146 : context->count_symlink = count_symlink;
3507 146 : context->count_dir = count_dir;
3508 :
3509 146 : return 0;
3510 : }
3511 :
3512 146 : static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
3513 : {
3514 : #if HAVE_MT_WRITE
3515 : int fail;
3516 : int first;
3517 : #else
3518 : STREAM* f;
3519 : unsigned count_content;
3520 : unsigned k;
3521 : struct state_write_thread_context* context;
3522 : void* retval;
3523 : #endif
3524 : tommy_node* i;
3525 : block_off_t blockmax;
3526 : time_t info_oldest;
3527 : time_t info_now;
3528 : int info_has_rehash;
3529 : int mapping_idx;
3530 : block_off_t idx;
3531 : uint32_t crc;
3532 : unsigned count_file;
3533 : unsigned count_hardlink;
3534 : unsigned count_symlink;
3535 : unsigned count_dir;
3536 :
3537 : /* blocks of all array */
3538 146 : blockmax = parity_allocated_size(state);
3539 :
3540 : /* check the file-system on all disks */
3541 146 : state_fscheck(state, "before write");
3542 :
3543 : /* clear the info for unused blocks */
3544 : /* and get some other info */
3545 146 : info_oldest = 0; /* oldest time in info */
3546 146 : info_now = time(0); /* get the present time */
3547 146 : info_has_rehash = 0; /* if there is a rehash info */
3548 735082 : for (idx = 0; idx < blockmax; ++idx) {
3549 : /* if the position is used */
3550 734936 : if (fs_position_is_required(state, idx)) {
3551 734936 : snapraid_info info = info_get(&state->infoarr, idx);
3552 :
3553 : /* only if there is some info to store */
3554 734936 : if (info) {
3555 690451 : time_t info_time = info_get_time(info);
3556 :
3557 690451 : if (!info_oldest || info_time < info_oldest)
3558 146 : info_oldest = info_time;
3559 :
3560 690451 : if (info_get_rehash(info))
3561 13822 : info_has_rehash = 1;
3562 : }
3563 : } else {
3564 : /* clear any previous info */
3565 0 : info_set(&state->infoarr, idx, 0);
3566 :
3567 : /* and clear any deleted blocks */
3568 0 : fs_position_clear_deleted(state, idx);
3569 : }
3570 : }
3571 :
3572 : /* map disks */
3573 146 : mapping_idx = 0;
3574 1022 : for (i = state->maplist; i != 0; i = i->next) {
3575 876 : struct snapraid_map* map = i->data;
3576 : struct snapraid_disk* disk;
3577 :
3578 : /* find the disk for this mapping */
3579 876 : disk = find_disk_by_name(state, map->name);
3580 876 : if (!disk) {
3581 : /* LCOV_EXCL_START */
3582 : log_fatal("Internal inconsistency: Unmapped disk '%s'\n", map->name);
3583 : os_abort();
3584 : /* LCOV_EXCL_STOP */
3585 : }
3586 :
3587 : /* save the mapping only for not empty disks */
3588 876 : if (!fs_is_empty(disk, blockmax)) {
3589 : /* assign the mapping index used to identify disks */
3590 784 : disk->mapping_idx = mapping_idx;
3591 784 : ++mapping_idx;
3592 : } else {
3593 : /* mark the disk as without mapping */
3594 92 : disk->mapping_idx = -1;
3595 : }
3596 : }
3597 :
3598 : #if HAVE_MT_WRITE
3599 : /* start all writing threads */
3600 : i = tommy_list_head(&state->contentlist);
3601 : while (i) {
3602 : struct snapraid_content* content = i->data;
3603 : struct state_write_thread_context* context;
3604 : char tmp[PATH_MAX];
3605 : STREAM* f;
3606 :
3607 : msg_progress("Saving state to %s...\n", content->content);
3608 :
3609 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3610 :
3611 : /* ensure to delete a previous stale file */
3612 : if (remove(tmp) != 0) {
3613 : if (errno != ENOENT) {
3614 : /* LCOV_EXCL_START */
3615 : log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
3616 : exit(EXIT_FAILURE);
3617 : /* LCOV_EXCL_STOP */
3618 : }
3619 : }
3620 :
3621 : f = sopen_write(tmp);
3622 : if (f == 0) {
3623 : /* LCOV_EXCL_START */
3624 : log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3625 : exit(EXIT_FAILURE);
3626 : /* LCOV_EXCL_STOP */
3627 : }
3628 :
3629 : /* allocate the thread context */
3630 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3631 : content->context = context;
3632 :
3633 : /* initialize */
3634 : context->state = state;
3635 : context->blockmax = blockmax;
3636 : context->info_oldest = info_oldest;
3637 : context->info_now = info_now;
3638 : context->info_has_rehash = info_has_rehash;
3639 : context->f = f;
3640 :
3641 : thread_create(&context->thread, state_write_thread, context);
3642 :
3643 : i = i->next;
3644 : }
3645 :
3646 : /* join all thread */
3647 : fail = 0;
3648 : first = 1;
3649 : crc = 0;
3650 : count_file = 0;
3651 : count_hardlink = 0;
3652 : count_symlink = 0;
3653 : count_dir = 0;
3654 : i = tommy_list_head(&state->contentlist);
3655 : while (i) {
3656 : struct snapraid_content* content = i->data;
3657 : struct state_write_thread_context* context = content->context;
3658 : void* retval;
3659 :
3660 : thread_join(context->thread, &retval);
3661 :
3662 : if (retval) {
3663 : /* LCOV_EXCL_START */
3664 : fail = 1;
3665 : /* LCOV_EXCL_STOP */
3666 : } else {
3667 : STREAM* f = context->f;
3668 :
3669 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3670 : /* than even in a system crash event we have one valid copy of the file. */
3671 : if (sflush(f) != 0) {
3672 : /* LCOV_EXCL_START */
3673 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3674 : exit(EXIT_FAILURE);
3675 : /* LCOV_EXCL_STOP */
3676 : }
3677 :
3678 : #if HAVE_FSYNC
3679 : if (ssync(f) != 0) {
3680 : /* LCOV_EXCL_START */
3681 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3682 : exit(EXIT_FAILURE);
3683 : /* LCOV_EXCL_STOP */
3684 : }
3685 : #endif
3686 :
3687 : if (sclose(f) != 0) {
3688 : /* LCOV_EXCL_START */
3689 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3690 : exit(EXIT_FAILURE);
3691 : /* LCOV_EXCL_STOP */
3692 : }
3693 :
3694 : if (first) {
3695 : first = 0;
3696 : crc = context->crc;
3697 : count_file = context->count_file;
3698 : count_hardlink = context->count_hardlink;
3699 : count_symlink = context->count_symlink;
3700 : count_dir = context->count_dir;
3701 : } else {
3702 : if (crc != context->crc) {
3703 : /* LCOV_EXCL_START */
3704 : log_fatal("Different CRCs writing content streams.\n");
3705 : log_fatal("DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
3706 : log_fatal("Try running a memory test like http://www.memtest86.com/\n");
3707 : exit(EXIT_FAILURE);
3708 : /* LCOV_EXCL_STOP */
3709 : }
3710 : }
3711 : }
3712 :
3713 : free(context);
3714 :
3715 : i = i->next;
3716 : }
3717 :
3718 : /* abort on failure */
3719 : if (fail) {
3720 : /* LCOV_EXCL_START */
3721 : exit(EXIT_FAILURE);
3722 : /* LCOV_EXCL_STOP */
3723 : }
3724 : #else
3725 : /* count the content files */
3726 146 : count_content = 0;
3727 146 : i = tommy_list_head(&state->contentlist);
3728 1123 : while (i) {
3729 977 : struct snapraid_content* content = i->data;
3730 977 : msg_progress("Saving state to %s...\n", content->content);
3731 977 : ++count_content;
3732 977 : i = i->next;
3733 : }
3734 :
3735 : /* open all the content files */
3736 146 : f = sopen_multi_write(count_content);
3737 146 : if (!f) {
3738 : /* LCOV_EXCL_START */
3739 : log_fatal("Error opening the content files.\n");
3740 : exit(EXIT_FAILURE);
3741 : /* LCOV_EXCL_STOP */
3742 : }
3743 :
3744 146 : k = 0;
3745 146 : i = tommy_list_head(&state->contentlist);
3746 1123 : while (i) {
3747 977 : struct snapraid_content* content = i->data;
3748 : char tmp[PATH_MAX];
3749 977 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
3750 :
3751 : /* ensure to delete a previous stale file */
3752 977 : if (remove(tmp) != 0) {
3753 977 : if (errno != ENOENT) {
3754 : /* LCOV_EXCL_START */
3755 : log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
3756 : exit(EXIT_FAILURE);
3757 : /* LCOV_EXCL_STOP */
3758 : }
3759 : }
3760 :
3761 977 : if (sopen_multi_file(f, k, tmp) != 0) {
3762 : /* LCOV_EXCL_START */
3763 : log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
3764 : exit(EXIT_FAILURE);
3765 : /* LCOV_EXCL_STOP */
3766 : }
3767 :
3768 977 : ++k;
3769 977 : i = i->next;
3770 : }
3771 :
3772 : /* allocate the thread context */
3773 146 : context = malloc_nofail(sizeof(struct state_write_thread_context));
3774 :
3775 : /* initialize */
3776 146 : context->state = state;
3777 146 : context->blockmax = blockmax;
3778 146 : context->info_oldest = info_oldest;
3779 146 : context->info_now = info_now;
3780 146 : context->info_has_rehash = info_has_rehash;
3781 146 : context->f = f;
3782 :
3783 146 : retval = state_write_thread(context);
3784 :
3785 : /* abort on failure */
3786 146 : if (retval) {
3787 : /* LCOV_EXCL_START */
3788 : exit(EXIT_FAILURE);
3789 : /* LCOV_EXCL_STOP */
3790 : }
3791 :
3792 : /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
3793 : /* than even in a system crash event we have one valid copy of the file. */
3794 146 : if (sflush(f) != 0) {
3795 : /* LCOV_EXCL_START */
3796 : log_fatal("Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
3797 : exit(EXIT_FAILURE);
3798 : /* LCOV_EXCL_STOP */
3799 : }
3800 :
3801 : #if HAVE_FSYNC
3802 146 : if (ssync(f) != 0) {
3803 : /* LCOV_EXCL_START */
3804 : log_fatal("Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
3805 : exit(EXIT_FAILURE);
3806 : /* LCOV_EXCL_STOP */
3807 : }
3808 : #endif
3809 :
3810 146 : if (sclose(f) != 0) {
3811 : /* LCOV_EXCL_START */
3812 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
3813 : exit(EXIT_FAILURE);
3814 : /* LCOV_EXCL_STOP */
3815 : }
3816 :
3817 146 : crc = context->crc;
3818 146 : count_file = context->count_file;
3819 146 : count_hardlink = context->count_hardlink;
3820 146 : count_symlink = context->count_symlink;
3821 146 : count_dir = context->count_dir;
3822 :
3823 146 : free(context);
3824 : #endif
3825 :
3826 146 : msg_verbose("%8u files\n", count_file);
3827 146 : msg_verbose("%8u hardlinks\n", count_hardlink);
3828 146 : msg_verbose("%8u symlinks\n", count_symlink);
3829 146 : msg_verbose("%8u empty dirs\n", count_dir);
3830 :
3831 146 : *out_crc = crc;
3832 146 : }
3833 :
3834 262 : void state_read(struct snapraid_state* state)
3835 : {
3836 : STREAM* f;
3837 : char path[PATH_MAX];
3838 : struct stat st;
3839 : tommy_node* node;
3840 : int ret;
3841 : int c;
3842 :
3843 : /* iterate over all the available content files and load the first one present */
3844 262 : f = 0;
3845 262 : node = tommy_list_head(&state->contentlist);
3846 312 : while (node) {
3847 305 : struct snapraid_content* content = node->data;
3848 305 : pathcpy(path, sizeof(path), content->content);
3849 :
3850 305 : if (!state->no_conf) {
3851 304 : log_tag("content:%s\n", path);
3852 304 : log_flush();
3853 : }
3854 305 : msg_progress("Loading state from %s...\n", path);
3855 :
3856 305 : f = sopen_read(path);
3857 305 : if (f != 0) {
3858 : /* if opened stop the search */
3859 255 : break;
3860 : } else {
3861 : /* if it's real error of an existing file, abort */
3862 50 : if (errno != ENOENT) {
3863 : /* LCOV_EXCL_START */
3864 : log_fatal("Error opening the content file '%s'. %s.\n", path, strerror(errno));
3865 : exit(EXIT_FAILURE);
3866 : /* LCOV_EXCL_STOP */
3867 : }
3868 :
3869 : /* otherwise continue */
3870 50 : if (node->next) {
3871 43 : log_fatal("WARNING! Content file '%s' not found, attempting with another copy...\n", path);
3872 :
3873 : /* ensure to rewrite all the content files */
3874 43 : state->need_write = 1;
3875 : }
3876 : }
3877 :
3878 : /* next content file */
3879 50 : node = node->next;
3880 : }
3881 :
3882 : /* if not found, assume empty */
3883 262 : if (!f) {
3884 7 : log_fatal("No content file found. Assuming empty.\n");
3885 :
3886 : /* create the initial mapping */
3887 7 : state_map(state);
3888 7 : return;
3889 : }
3890 :
3891 : /* get the stat of the content file */
3892 255 : ret = fstat(shandle(f), &st);
3893 255 : if (ret != 0) {
3894 : /* LCOV_EXCL_START */
3895 : log_fatal("Error stating the content file '%s'. %s.\n", path, strerror(errno));
3896 : exit(EXIT_FAILURE);
3897 : /* LCOV_EXCL_STOP */
3898 : }
3899 :
3900 : /* go further to check other content files */
3901 1709 : while (node) {
3902 : char other_path[PATH_MAX];
3903 : struct stat other_st;
3904 1454 : struct snapraid_content* content = node->data;
3905 1454 : pathcpy(other_path, sizeof(other_path), content->content);
3906 :
3907 1454 : ret = stat(other_path, &other_st);
3908 1454 : if (ret != 0) {
3909 : /* allow missing content files, but not any other kind of error */
3910 1 : if (errno != ENOENT) {
3911 : /* LCOV_EXCL_START */
3912 : log_fatal("Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
3913 : exit(EXIT_FAILURE);
3914 : /* LCOV_EXCL_STOP */
3915 : }
3916 :
3917 : /* ensure to rewrite all the content files */
3918 1 : state->need_write = 1;
3919 : } else {
3920 : /* if the size is different */
3921 1453 : if (other_st.st_size != st.st_size) {
3922 39 : log_fatal("WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
3923 39 : log_fatal("Likely one of the two is broken!\n");
3924 :
3925 : /* ensure to rewrite all the content files */
3926 39 : state->need_write = 1;
3927 : }
3928 : }
3929 :
3930 : /* next content file */
3931 1454 : node = node->next;
3932 : }
3933 :
3934 : /* start with a undefined default. */
3935 : /* it's for compatibility with version 1.0 where MD5 was implicit. */
3936 255 : state->hash = HASH_UNDEFINED;
3937 :
3938 : /* start with a zero seed, it was the default in old versions */
3939 255 : memset(state->hashseed, 0, HASH_MAX);
3940 :
3941 : /* previous hash, start with an undefined value */
3942 255 : state->prevhash = HASH_UNDEFINED;
3943 :
3944 : /* intentionally not set the prevhashseed, if used valgrind will warn about it */
3945 :
3946 : /* get the first char to detect the file type */
3947 255 : c = sgetc(f);
3948 255 : sungetc(c, f);
3949 :
3950 : /* guess the file type from the first char */
3951 255 : if (c == 'S') {
3952 255 : state_read_content(state, path, f);
3953 : } else {
3954 : /* LCOV_EXCL_START */
3955 : log_fatal("From SnapRAID v9.0 the text content file is not supported anymore.\n");
3956 : log_fatal("You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
3957 : exit(EXIT_FAILURE);
3958 : /* LCOV_EXCL_STOP */
3959 : }
3960 :
3961 254 : sclose(f);
3962 :
3963 254 : if (state->hash == HASH_UNDEFINED) {
3964 : /* LCOV_EXCL_START */
3965 : log_fatal("The checksum to use is not specified.\n");
3966 : log_fatal("This happens because you are likely upgrading from SnapRAID 1.0.\n");
3967 : log_fatal("To use a new SnapRAID you must restart from scratch,\n");
3968 : log_fatal("deleting all the content and parity files.\n");
3969 : exit(EXIT_FAILURE);
3970 : /* LCOV_EXCL_STOP */
3971 : }
3972 :
3973 : /* update the mapping */
3974 254 : state_map(state);
3975 :
3976 254 : state_content_check(state, path);
3977 :
3978 : /* mark that we read the content file, and it passed all the checks */
3979 254 : state->checked_read = 1;
3980 : }
3981 :
3982 : struct state_verify_thread_context {
3983 : struct snapraid_state* state;
3984 : struct snapraid_content* content;
3985 : #if HAVE_MT_VERIFY
3986 : thread_id_t thread;
3987 : #else
3988 : void* retval;
3989 : #endif
3990 : /* input */
3991 : uint32_t crc;
3992 : STREAM* f;
3993 : };
3994 :
3995 977 : static void* state_verify_thread(void* arg)
3996 : {
3997 977 : struct state_verify_thread_context* context = arg;
3998 977 : struct snapraid_content* content = context->content;
3999 977 : STREAM* f = context->f;
4000 : unsigned char buf[4];
4001 : uint32_t crc_stored;
4002 : uint32_t crc_computed;
4003 : uint64_t start;
4004 :
4005 977 : start = tick_ms();
4006 :
4007 977 : if (sdeplete(f, buf) != 0) {
4008 : /* LCOV_EXCL_START */
4009 : log_fatal("Failed to flush content file '%s'. %s.\n", serrorfile(f), strerror(errno));
4010 : return context;
4011 : /* LCOV_EXCL_STOP */
4012 : }
4013 :
4014 : /* get the stored crc from the last four bytes */
4015 977 : crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
4016 :
4017 977 : if (crc_stored != context->crc) {
4018 : /* LCOV_EXCL_START */
4019 : log_fatal("DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
4020 : return context;
4021 : /* LCOV_EXCL_STOP */
4022 : }
4023 :
4024 : /* get the computed crc */
4025 977 : crc_computed = scrc(f);
4026 :
4027 : /* adjust the stored crc to include itself */
4028 977 : crc_stored = crc32c(crc_stored, buf, 4);
4029 :
4030 977 : if (crc_computed != crc_stored) {
4031 : /* LCOV_EXCL_START */
4032 : log_fatal("DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
4033 : return context;
4034 : /* LCOV_EXCL_STOP */
4035 : }
4036 :
4037 977 : msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (tick_ms() - start) / 1000);
4038 :
4039 977 : return 0;
4040 : }
4041 :
4042 146 : static void state_verify_content(struct snapraid_state* state, uint32_t crc)
4043 : {
4044 : tommy_node* i;
4045 : int fail;
4046 :
4047 146 : msg_progress("Verifying...\n");
4048 :
4049 : /* start all reading threads */
4050 146 : i = tommy_list_head(&state->contentlist);
4051 1123 : while (i) {
4052 977 : struct snapraid_content* content = i->data;
4053 : struct state_verify_thread_context* context;
4054 : char tmp[PATH_MAX];
4055 : STREAM* f;
4056 :
4057 977 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4058 977 : f = sopen_read(tmp);
4059 977 : if (f == 0) {
4060 : /* LCOV_EXCL_START */
4061 : log_fatal("Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
4062 : exit(EXIT_FAILURE);
4063 : /* LCOV_EXCL_STOP */
4064 : }
4065 :
4066 : /* allocate the thread context */
4067 977 : context = malloc_nofail(sizeof(struct state_verify_thread_context));
4068 977 : content->context = context;
4069 :
4070 : /* initialize */
4071 977 : context->state = state;
4072 977 : context->content = content;
4073 977 : context->crc = crc;
4074 977 : context->f = f;
4075 :
4076 : #if HAVE_MT_VERIFY
4077 977 : thread_create(&context->thread, state_verify_thread, context);
4078 : #else
4079 : context->retval = state_verify_thread(context);
4080 : #endif
4081 :
4082 977 : i = i->next;
4083 : }
4084 :
4085 : /* join all thread */
4086 146 : fail = 0;
4087 146 : i = tommy_list_head(&state->contentlist);
4088 1123 : while (i) {
4089 977 : struct snapraid_content* content = i->data;
4090 977 : struct state_verify_thread_context* context = content->context;
4091 : void* retval;
4092 :
4093 : #if HAVE_MT_VERIFY
4094 977 : thread_join(context->thread, &retval);
4095 : #else
4096 : retval = context->retval;
4097 : #endif
4098 977 : if (retval) {
4099 : /* LCOV_EXCL_START */
4100 : fail = 1;
4101 : /* LCOV_EXCL_STOP */
4102 : } else {
4103 977 : STREAM* f = context->f;
4104 :
4105 977 : if (sclose(f) != 0) {
4106 : /* LCOV_EXCL_START */
4107 : log_fatal("Error closing the content file. %s.\n", strerror(errno));
4108 : exit(EXIT_FAILURE);
4109 : /* LCOV_EXCL_STOP */
4110 : }
4111 : }
4112 :
4113 977 : free(context);
4114 :
4115 977 : i = i->next;
4116 : }
4117 :
4118 : /* abort on failure */
4119 146 : if (fail) {
4120 : /* LCOV_EXCL_START */
4121 : exit(EXIT_FAILURE);
4122 : /* LCOV_EXCL_STOP */
4123 : }
4124 146 : }
4125 :
4126 146 : static void state_rename_content(struct snapraid_state* state)
4127 : {
4128 : tommy_node* i;
4129 :
4130 : #if defined(_linux) /* this sequence is linux specific */
4131 : i = tommy_list_head(&state->contentlist);
4132 : while (i) {
4133 : struct snapraid_content* content = i->data;
4134 : char tmp[PATH_MAX];
4135 : char dir[PATH_MAX];
4136 : char* slash;
4137 : int handle;
4138 :
4139 : pathcpy(dir, sizeof(dir), content->content);
4140 :
4141 : slash = strrchr(tmp, '/');
4142 : if (slash)
4143 : *slash = 0;
4144 : else
4145 : pathcpy(dir, sizeof(dir), ".");
4146 :
4147 : /* open the directory to get the handle */
4148 : handle = open(dir, O_RDONLY | O_DIRECTORY);
4149 : if (handle < 0) {
4150 : /* LCOV_EXCL_START */
4151 : log_fatal("Error opening the directory '%s'. %s.\n", dir, strerror(errno));
4152 : exit(EXIT_FAILURE);
4153 : /* LCOV_EXCL_STOP */
4154 : }
4155 :
4156 : /* now rename the just written copy with the correct name */
4157 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4158 : if (rename(tmp, content->content) != 0) {
4159 : /* LCOV_EXCL_START */
4160 : log_fatal("Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
4161 : exit(EXIT_FAILURE);
4162 : /* LCOV_EXCL_STOP */
4163 : }
4164 :
4165 : /* sync the directory */
4166 : if (fsync(handle) != 0) {
4167 : /* LCOV_EXCL_START */
4168 : log_fatal("Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
4169 : exit(EXIT_FAILURE);
4170 : /* LCOV_EXCL_STOP */
4171 : }
4172 :
4173 : if (close(handle) != 0) {
4174 : /* LCOV_EXCL_START */
4175 : log_fatal("Error closing the directory '%s'. %s.\n", dir, strerror(errno));
4176 : exit(EXIT_FAILURE);
4177 : /* LCOV_EXCL_STOP */
4178 : }
4179 :
4180 : i = i->next;
4181 : }
4182 : #else
4183 146 : i = tommy_list_head(&state->contentlist);
4184 1123 : while (i) {
4185 977 : struct snapraid_content* content = i->data;
4186 : char tmp[PATH_MAX];
4187 :
4188 : /* now renames the just written copy with the correct name */
4189 977 : pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
4190 977 : if (rename(tmp, content->content) != 0) {
4191 : /* LCOV_EXCL_START */
4192 : log_fatal("Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
4193 : exit(EXIT_FAILURE);
4194 : /* LCOV_EXCL_STOP */
4195 : }
4196 :
4197 977 : i = i->next;
4198 : }
4199 : #endif
4200 146 : }
4201 :
4202 146 : void state_write(struct snapraid_state* state)
4203 : {
4204 : uint32_t crc;
4205 :
4206 : /* write all the content files */
4207 146 : state_write_content(state, &crc);
4208 :
4209 : /* verify the just written files */
4210 146 : state_verify_content(state, crc);
4211 :
4212 : /* rename the new files, over the old ones */
4213 146 : state_rename_content(state);
4214 :
4215 146 : state->need_write = 0; /* no write needed anymore */
4216 146 : state->checked_read = 0; /* what we wrote is not checked in read */
4217 146 : state->written = 1;
4218 146 : }
4219 :
4220 123 : void state_skip(struct snapraid_state* state)
4221 : {
4222 : tommy_node* i;
4223 :
4224 : /* for each disk */
4225 857 : for (i = state->disklist; i != 0; i = i->next) {
4226 : tommy_node* j;
4227 734 : struct snapraid_disk* disk = i->data;
4228 :
4229 734 : if (!disk->skip_access)
4230 733 : continue;
4231 :
4232 : /* for each file */
4233 1885 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4234 1884 : struct snapraid_file* file = j->data;
4235 1884 : file_flag_set(file, FILE_IS_EXCLUDED);
4236 : }
4237 :
4238 : /* for each link */
4239 57 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4240 56 : struct snapraid_link* slink = j->data;
4241 56 : link_flag_set(slink, FILE_IS_EXCLUDED);
4242 : }
4243 :
4244 : /* for each dir */
4245 1 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4246 0 : struct snapraid_dir* dir = j->data;
4247 0 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4248 : }
4249 : }
4250 123 : }
4251 :
4252 123 : void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
4253 : {
4254 : tommy_node* i;
4255 : unsigned l;
4256 :
4257 : /* if no filter, include all */
4258 123 : if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
4259 114 : return;
4260 :
4261 9 : msg_progress("Selecting...\n");
4262 :
4263 17 : for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
4264 8 : struct snapraid_filter* filter = i->data;
4265 8 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
4266 : }
4267 12 : for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
4268 3 : struct snapraid_filter* filter = i->data;
4269 3 : msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
4270 : }
4271 9 : if (filter_missing)
4272 2 : msg_verbose("\t<missing>\n");
4273 9 : if (filter_error)
4274 2 : msg_verbose("\t<error>\n");
4275 :
4276 : /* for each disk */
4277 63 : for (i = state->disklist; i != 0; i = i->next) {
4278 : tommy_node* j;
4279 54 : struct snapraid_disk* disk = i->data;
4280 :
4281 : /* if we filter for presence, we have to access the disk, so better to print something */
4282 54 : if (filter_missing)
4283 12 : msg_progress("Scanning disk %s...\n", disk->name);
4284 :
4285 : /* for each file */
4286 133944 : for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
4287 133890 : struct snapraid_file* file = j->data;
4288 :
4289 133890 : if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
4290 80982 : || filter_path(filterlist_file, 0, disk->name, file->sub) != 0
4291 78138 : || filter_existence(filter_missing, disk->dir, file->sub) != 0
4292 38646 : || filter_correctness(filter_error, &state->infoarr, disk, file) != 0
4293 : ) {
4294 109494 : file_flag_set(file, FILE_IS_EXCLUDED);
4295 : }
4296 : }
4297 :
4298 : /* for each link */
4299 4414 : for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
4300 4360 : struct snapraid_link* slink = j->data;
4301 :
4302 4360 : if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
4303 2641 : || filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
4304 2536 : || filter_existence(filter_missing, disk->dir, slink->sub) != 0
4305 : ) {
4306 3221 : link_flag_set(slink, FILE_IS_EXCLUDED);
4307 : }
4308 : }
4309 :
4310 : /* for each empty dir */
4311 82 : for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
4312 28 : struct snapraid_dir* dir = j->data;
4313 :
4314 28 : if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
4315 17 : || filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
4316 17 : || filter_existence(filter_missing, disk->dir, dir->sub) != 0
4317 : ) {
4318 19 : dir_flag_set(dir, FILE_IS_EXCLUDED);
4319 : }
4320 : }
4321 : }
4322 :
4323 : /* if we are filtering by disk, exclude any parity not explicitly included */
4324 9 : if (!tommy_list_empty(filterlist_disk)) {
4325 : /* for each parity disk */
4326 20 : for (l = 0; l < state->level; ++l) {
4327 : /* check if the parity is excluded by name */
4328 16 : if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
4329 : /* excluded the parity from further operation */
4330 11 : state->parity[l].is_excluded_by_filter = 1;
4331 : }
4332 : }
4333 : } else {
4334 : /* if we are filtering by file, exclude all parity */
4335 5 : if (filter_missing || !tommy_list_empty(filterlist_file)) {
4336 : /* for each parity disk */
4337 16 : for (l = 0; l < state->level; ++l) {
4338 13 : state->parity[l].is_excluded_by_filter = 1;
4339 : }
4340 : }
4341 : }
4342 : }
4343 :
4344 222 : int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
4345 : {
4346 : time_t now;
4347 :
4348 222 : if (state->opt.gui) {
4349 5 : log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
4350 5 : log_flush();
4351 : }
4352 :
4353 222 : now = time(0);
4354 :
4355 222 : state->progress_whole_start = now;
4356 222 : state->progress_tick = 0;
4357 222 : state->progress_ptr = 0;
4358 222 : state->progress_wasted = 0;
4359 :
4360 : /* start of thermal control */
4361 222 : if (!state_thermal_begin(state, now))
4362 0 : return 0;
4363 :
4364 : /* stop if requested */
4365 222 : if (global_interrupt) {
4366 : /* LCOV_EXCL_START */
4367 : if (!state->opt.gui) {
4368 : msg_status("Not starting for interruption\n");
4369 : }
4370 : log_tag("sigint:0: SIGINT received\n");
4371 : log_flush();
4372 : return 0;
4373 : /* LCOV_EXCL_STOP */
4374 : }
4375 :
4376 222 : return 1;
4377 : }
4378 :
4379 222 : void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize, const char* msg)
4380 : {
4381 222 : if (state->opt.gui) {
4382 5 : log_tag("run:end\n");
4383 5 : log_flush();
4384 217 : } else if (countmax == 0) {
4385 21 : if (state->need_write || state->written) {
4386 16 : msg_status("100%% completed\n");
4387 : } else {
4388 5 : msg_status("%s", msg);
4389 : }
4390 : } else {
4391 : time_t now;
4392 : time_t elapsed;
4393 :
4394 196 : unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
4395 :
4396 196 : now = time(0);
4397 :
4398 196 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4399 :
4400 196 : msg_bar("%u%% completed, %u MB accessed", muldiv(countpos, 100, countmax), countsize_MB);
4401 :
4402 196 : msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
4403 :
4404 196 : msg_bar("\n");
4405 196 : msg_flush();
4406 : }
4407 222 : }
4408 :
4409 1 : void state_progress_stop(struct snapraid_state* state)
4410 : {
4411 : time_t now;
4412 :
4413 1 : now = time(0);
4414 :
4415 1 : state->progress_interruption = now;
4416 1 : }
4417 :
4418 1 : void state_progress_restart(struct snapraid_state* state)
4419 : {
4420 : time_t now;
4421 :
4422 1 : now = time(0);
4423 :
4424 : /* reset the progress counter */
4425 1 : state->progress_tick = 0;
4426 1 : state->progress_ptr = 0;
4427 :
4428 1 : if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
4429 1 : state->progress_wasted += now - state->progress_interruption;
4430 1 : }
4431 :
4432 : /**
4433 : * Set the latest tick data in the progress vector.
4434 : */
4435 470 : static void state_progress_latest(struct snapraid_state* state)
4436 : {
4437 : tommy_node* i;
4438 : unsigned l;
4439 :
4440 470 : state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
4441 470 : state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
4442 470 : state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
4443 470 : state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
4444 470 : state->progress_tick_io[state->progress_ptr] = state->tick_io;
4445 3281 : for (i = state->disklist; i != 0; i = i->next) {
4446 2811 : struct snapraid_disk* disk = i->data;
4447 2811 : disk->progress_tick[state->progress_ptr] = disk->tick;
4448 : }
4449 2745 : for (l = 0; l < state->level; ++l)
4450 2275 : state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
4451 470 : }
4452 :
4453 : /**
4454 : * Number of columns
4455 : */
4456 : #define GRAPH_COLUMN 78
4457 :
4458 : /**
4459 : * Get the reference value, 0 if index is PROGRESS_MAX
4460 : */
4461 : #define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
4462 :
4463 0 : static struct snapraid_thermal* state_thermal_find(struct snapraid_state* state, const char* name)
4464 : {
4465 : tommy_node* i;
4466 0 : struct snapraid_thermal* found = 0;
4467 :
4468 0 : for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
4469 0 : struct snapraid_thermal* thermal = i->data;
4470 0 : if (strcmp(thermal->name, name) == 0) {
4471 0 : if (found == 0
4472 : /* if multiple matches, return the one with highest temperature */
4473 0 : || found->latest_temperature < thermal->latest_temperature
4474 : /* or, if the temperature is equal, the one with the higher r_squared */
4475 0 : || (found->latest_temperature == thermal->latest_temperature && found->params.r_squared < thermal->params.r_squared)
4476 : ) {
4477 0 : found = thermal;
4478 : }
4479 : }
4480 : }
4481 :
4482 0 : return found;
4483 : }
4484 :
4485 : #define THERMAL_PAD " "
4486 :
4487 1 : static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
4488 : {
4489 : uint64_t v;
4490 : uint64_t tick_total;
4491 : unsigned bar;
4492 : tommy_node* i;
4493 : unsigned l;
4494 : size_t pad;
4495 : size_t pre;
4496 :
4497 1 : tick_total = 0;
4498 :
4499 1 : tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4500 1 : tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4501 1 : tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4502 1 : tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4503 1 : tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
4504 1 : if (!tick_total)
4505 0 : return;
4506 :
4507 1 : pad = 4;
4508 7 : for (i = state->disklist; i != 0; i = i->next) {
4509 6 : struct snapraid_disk* disk = i->data;
4510 6 : size_t len = strlen(disk->name);
4511 6 : if (pad < len)
4512 1 : pad = len;
4513 : }
4514 7 : for (l = 0; l < state->level; ++l) {
4515 6 : size_t len = strlen(lev_config_name(l));
4516 6 : if (pad < len)
4517 2 : pad = len;
4518 : }
4519 :
4520 : /* extra space */
4521 1 : pad += 1;
4522 :
4523 1 : pre = 4;
4524 1 : if (pad + pre + 30 < GRAPH_COLUMN)
4525 1 : bar = GRAPH_COLUMN - pad - pre;
4526 : else
4527 0 : bar = 30; /* at least a bar of 30 */
4528 :
4529 1 : if (io) {
4530 0 : const char* legend = "cached blocks (instant, more is better)";
4531 :
4532 : /* refresh the cached blocks info */
4533 0 : io_refresh(io);
4534 :
4535 0 : printf("\n");
4536 :
4537 0 : for (i = state->disklist; i != 0; i = i->next) {
4538 0 : struct snapraid_disk* disk = i->data;
4539 0 : v = disk->cached_blocks;
4540 0 : printr(disk->name, pad);
4541 0 : printf("%4" PRIu64 " | ", v);
4542 :
4543 0 : if (disk->progress_file && disk->progress_file->sub)
4544 0 : printf("%s", disk->progress_file->sub);
4545 : else
4546 0 : printf("-");
4547 :
4548 0 : printf("\n");
4549 : }
4550 :
4551 0 : for (l = 0; l < state->level; ++l) {
4552 0 : v = state->parity[l].cached_blocks;
4553 0 : printr(lev_config_name(l), pad);
4554 0 : printf("%4" PRIu64 " | ", v);
4555 0 : printc('o', v * bar / io->io_max);
4556 0 : printf("\n");
4557 : }
4558 :
4559 0 : printc(' ', pad);
4560 0 : printf(" |_");
4561 0 : printc('_', bar);
4562 0 : printf("\n");
4563 :
4564 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4565 0 : printf("%s", legend);
4566 0 : printf("\n");
4567 : }
4568 :
4569 1 : printf("\n");
4570 :
4571 1 : pre = 4;
4572 1 : if (state->thermal_temperature_limit != 0)
4573 0 : pre += strlen(THERMAL_PAD);
4574 1 : if (pad + pre + 30 < GRAPH_COLUMN)
4575 1 : bar = GRAPH_COLUMN - pad - pre;
4576 : else
4577 0 : bar = 30; /* at least a bar of 30 */
4578 :
4579 7 : for (i = state->disklist; i != 0; i = i->next) {
4580 6 : struct snapraid_disk* disk = i->data;
4581 :
4582 6 : v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
4583 6 : printr(disk->name, pad);
4584 :
4585 6 : if (state->thermal_temperature_limit != 0) {
4586 0 : struct snapraid_thermal* thermal = state_thermal_find(state, disk->name);
4587 0 : if (thermal) {
4588 0 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
4589 0 : printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
4590 : else
4591 0 : printf(" %2u ", thermal->latest_temperature);
4592 : } else {
4593 0 : printf(THERMAL_PAD);
4594 : }
4595 : }
4596 :
4597 6 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4598 6 : printc('*', v * bar / tick_total);
4599 6 : printf("\n");
4600 :
4601 : /* clear the file in progress */
4602 6 : disk->progress_file = 0;
4603 : }
4604 :
4605 7 : for (l = 0; l < state->level; ++l) {
4606 6 : v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
4607 6 : printr(lev_config_name(l), pad);
4608 :
4609 6 : if (state->thermal_temperature_limit != 0) {
4610 0 : struct snapraid_thermal* thermal = state_thermal_find(state, lev_config_name(l));
4611 0 : if (thermal) {
4612 0 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
4613 0 : printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
4614 : else
4615 0 : printf(" %2u ", thermal->latest_temperature);
4616 : } else {
4617 0 : printf(THERMAL_PAD);
4618 : }
4619 : }
4620 :
4621 6 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4622 6 : printc('*', v * bar / tick_total);
4623 6 : printf("\n");
4624 : }
4625 :
4626 1 : v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
4627 1 : printr("raid", pad);
4628 1 : if (state->thermal_temperature_limit != 0)
4629 0 : printf(THERMAL_PAD);
4630 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4631 1 : printc('*', v * bar / tick_total);
4632 1 : printf("\n");
4633 :
4634 1 : v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
4635 1 : printr("hash", pad);
4636 1 : if (state->thermal_temperature_limit != 0)
4637 0 : printf(THERMAL_PAD);
4638 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4639 1 : printc('*', v * bar / tick_total);
4640 1 : printf("\n");
4641 :
4642 1 : v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
4643 1 : printr("sched", pad);
4644 1 : if (state->thermal_temperature_limit != 0)
4645 0 : printf(THERMAL_PAD);
4646 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4647 1 : printc('*', v * bar / tick_total);
4648 1 : printf("\n");
4649 :
4650 1 : v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
4651 1 : printr("misc", pad);
4652 1 : if (state->thermal_temperature_limit != 0)
4653 0 : printf(THERMAL_PAD);
4654 1 : printf("%3u%% | ", muldiv(v, 100, tick_total));
4655 1 : printc('*', v * bar / tick_total);
4656 1 : printf("\n");
4657 :
4658 1 : printc(' ', pad);
4659 1 : if (state->thermal_temperature_limit != 0)
4660 0 : printf(THERMAL_PAD);
4661 1 : printf(" |_");
4662 1 : printc('_', bar);
4663 1 : printf("\n");
4664 :
4665 1 : if (oldest == PROGRESS_MAX) {
4666 1 : const char* legend = "wait time (total, less is better)";
4667 1 : if (state->thermal_temperature_limit != 0)
4668 0 : printf(THERMAL_PAD);
4669 1 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
4670 1 : printf("%s", legend);
4671 1 : printf("\n");
4672 : } else {
4673 0 : const char* legend_d = "wait time (last %d secs, less is better)";
4674 0 : if (state->thermal_temperature_limit != 0)
4675 0 : printf(THERMAL_PAD);
4676 0 : printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
4677 0 : printf(legend_d, PROGRESS_MAX);
4678 0 : printf("\n");
4679 : }
4680 :
4681 1 : printf("\n");
4682 : }
4683 :
4684 1073972 : int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
4685 : {
4686 : time_t now;
4687 : int pred;
4688 :
4689 1073972 : now = time(0);
4690 :
4691 : /* thermal measure */
4692 1073972 : if (now > state->thermal_latest + THERMAL_PERIOD_SECONDS || state->opt.fake_device) {
4693 2608 : state->thermal_latest = now;
4694 2608 : state_thermal(state, now);
4695 : }
4696 :
4697 : /* previous position */
4698 1073972 : pred = state->progress_ptr + PROGRESS_MAX - 1;
4699 1073972 : if (pred >= PROGRESS_MAX)
4700 1073774 : pred -= PROGRESS_MAX;
4701 :
4702 : /* if the previous measure is different */
4703 1073972 : if (state->progress_tick == 0
4704 1073774 : || state->progress_time[pred] != now
4705 : ) {
4706 : time_t elapsed;
4707 373 : unsigned out_perc = 0;
4708 373 : unsigned out_size_speed = 0;
4709 373 : unsigned out_block_speed = 0;
4710 373 : unsigned out_cpu = 0;
4711 373 : unsigned out_eta = 0;
4712 373 : int out_temperature = 0;
4713 373 : int out_steady = 0;
4714 373 : int out_computed = 0;
4715 :
4716 : /* store the new measure */
4717 373 : state->progress_time[state->progress_ptr] = now;
4718 373 : state->progress_pos[state->progress_ptr] = countpos;
4719 373 : state->progress_size[state->progress_ptr] = countsize;
4720 373 : state_progress_latest(state);
4721 :
4722 373 : elapsed = now - state->progress_whole_start - state->progress_wasted;
4723 :
4724 : /* completion percentage */
4725 373 : if (countmax)
4726 373 : out_perc = muldiv(countpos, 100, countmax);
4727 :
4728 : /* if we have at least 5 measures */
4729 373 : if (state->progress_tick >= 5
4730 : /* or if we are running in test mode, with at least one measure */
4731 372 : || (state->opt.force_progress && state->progress_tick >= 1)
4732 : ) {
4733 : int oldest;
4734 : int past;
4735 : time_t delta_time;
4736 : block_off_t delta_pos;
4737 : data_off_t delta_size;
4738 : uint64_t tick_cpu;
4739 : uint64_t tick_total;
4740 : uint64_t delta_tick_cpu;
4741 : uint64_t delta_tick_total;
4742 : uint64_t oldest_tick_cpu;
4743 : uint64_t oldest_tick_total;
4744 : tommy_node* i;
4745 :
4746 : /* number of past measures */
4747 175 : past = state->progress_tick;
4748 :
4749 : /* drop the oldest ones, to promptly */
4750 : /* skip the startup phase */
4751 175 : past -= past / 5;
4752 :
4753 : /* check how much we can go in the past */
4754 175 : if (past >= PROGRESS_MAX - 1) {
4755 : /* the vector is filled, so we are already in position */
4756 : /* to get the possible oldest one */
4757 0 : oldest = state->progress_ptr + 1;
4758 : } else {
4759 : /* go backward the number of positions selected */
4760 175 : oldest = state->progress_ptr + PROGRESS_MAX - past;
4761 : }
4762 175 : if (oldest >= PROGRESS_MAX)
4763 175 : oldest -= PROGRESS_MAX;
4764 :
4765 175 : tick_cpu = state->progress_tick_misc[state->progress_ptr]
4766 175 : + state->progress_tick_sched[state->progress_ptr]
4767 175 : + state->progress_tick_raid[state->progress_ptr]
4768 175 : + state->progress_tick_hash[state->progress_ptr];
4769 175 : tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
4770 :
4771 175 : oldest_tick_cpu = state->progress_tick_misc[oldest]
4772 175 : + state->progress_tick_sched[oldest]
4773 175 : + state->progress_tick_raid[oldest]
4774 175 : + state->progress_tick_hash[oldest];
4775 175 : oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
4776 :
4777 175 : delta_time = now - state->progress_time[oldest];
4778 175 : delta_pos = countpos - state->progress_pos[oldest];
4779 175 : delta_size = countsize - state->progress_size[oldest];
4780 175 : delta_tick_cpu = tick_cpu - oldest_tick_cpu;
4781 175 : delta_tick_total = tick_total - oldest_tick_total;
4782 :
4783 : /* estimate the speed in MB/s */
4784 175 : if (delta_time != 0)
4785 175 : out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
4786 :
4787 : /* estimate the speed in block/s */
4788 175 : if (delta_time != 0)
4789 175 : out_block_speed = (unsigned)(delta_pos / delta_time);
4790 :
4791 : /* estimate the cpu usage percentage */
4792 175 : if (delta_tick_total != 0)
4793 32 : out_cpu = muldiv(delta_tick_cpu, 100, delta_tick_total);
4794 :
4795 : /* estimate the remaining time in minutes */
4796 175 : if (delta_pos != 0)
4797 175 : out_eta = muldiv(countmax - countpos, delta_time, 60 * delta_pos);
4798 :
4799 175 : if (state->opt.force_stats) {
4800 0 : os_clear();
4801 0 : state_progress_graph(state, io, state->progress_ptr, oldest);
4802 : }
4803 :
4804 175 : if (state->thermal_temperature_limit != 0) {
4805 : /* get the max temperature */
4806 1 : out_temperature = 0;
4807 13 : for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
4808 12 : struct snapraid_thermal* thermal = i->data;
4809 12 : if (out_temperature < thermal->latest_temperature) {
4810 1 : out_temperature = thermal->latest_temperature;
4811 1 : if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
4812 1 : out_steady = thermal->params.t_steady;
4813 : else
4814 0 : out_steady = 0;
4815 : }
4816 : }
4817 : } else {
4818 174 : out_temperature = 0;
4819 174 : out_steady = 0;
4820 : }
4821 :
4822 : /* we have the output value */
4823 175 : out_computed = 1;
4824 : }
4825 :
4826 373 : if (state->opt.gui) {
4827 8 : log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_size_speed, out_cpu, (uint64_t)elapsed);
4828 8 : log_flush();
4829 : } else {
4830 365 : msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
4831 365 : if (out_computed) {
4832 172 : msg_bar(", %u MB/s", out_size_speed);
4833 172 : msg_bar(", %u stripe/s", out_block_speed);
4834 172 : msg_bar(", CPU %u%%", out_cpu);
4835 172 : if (out_temperature != 0) {
4836 1 : if (out_steady != 0)
4837 1 : msg_bar(", Tmax %u (%u)", out_temperature, out_steady);
4838 : else
4839 0 : msg_bar(", Tmax %u", out_temperature);
4840 : }
4841 172 : msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
4842 : }
4843 365 : msg_bar("\r");
4844 365 : msg_flush();
4845 : }
4846 :
4847 : /* next position to fill */
4848 373 : ++state->progress_ptr;
4849 373 : if (state->progress_ptr >= PROGRESS_MAX)
4850 0 : state->progress_ptr -= PROGRESS_MAX;
4851 :
4852 : /* one more measure */
4853 373 : ++state->progress_tick;
4854 : }
4855 :
4856 : /* stop if requested */
4857 1073972 : if (global_interrupt) {
4858 : /* LCOV_EXCL_START */
4859 : if (!state->opt.gui) {
4860 : log_fatal("\n");
4861 : log_fatal("Stopping for interruption at block %u\n", blockpos);
4862 : }
4863 : log_tag("sigint:%u: SIGINT received\n", blockpos);
4864 : log_flush();
4865 : return 1;
4866 : /* LCOV_EXCL_STOP */
4867 : }
4868 :
4869 1073972 : return 0;
4870 : }
4871 :
4872 105 : void state_usage_waste(struct snapraid_state* state)
4873 : {
4874 105 : uint64_t now = tick();
4875 :
4876 105 : state->tick_last = now;
4877 105 : }
4878 :
4879 1048425 : void state_usage_misc(struct snapraid_state* state)
4880 : {
4881 1048425 : uint64_t now = tick();
4882 1048425 : uint64_t delta = now - state->tick_last;
4883 :
4884 : /* increment the time spent in computations */
4885 1048425 : state->tick_misc += delta;
4886 :
4887 1048425 : state->tick_last = now;
4888 1048425 : }
4889 :
4890 147746 : void state_usage_sched(struct snapraid_state* state)
4891 : {
4892 147746 : uint64_t now = tick();
4893 147746 : uint64_t delta = now - state->tick_last;
4894 :
4895 : /* increment the time spent in computations */
4896 147746 : state->tick_sched += delta;
4897 :
4898 147746 : state->tick_last = now;
4899 147746 : }
4900 :
4901 137176 : void state_usage_raid(struct snapraid_state* state)
4902 : {
4903 137176 : uint64_t now = tick();
4904 137176 : uint64_t delta = now - state->tick_last;
4905 :
4906 : /* increment the time spent in computations */
4907 137176 : state->tick_raid += delta;
4908 :
4909 137176 : state->tick_last = now;
4910 137176 : }
4911 :
4912 749784 : void state_usage_hash(struct snapraid_state* state)
4913 : {
4914 749784 : uint64_t now = tick();
4915 749784 : uint64_t delta = now - state->tick_last;
4916 :
4917 : /* increment the time spent in computations */
4918 749784 : state->tick_hash += delta;
4919 :
4920 749784 : state->tick_last = now;
4921 749784 : }
4922 :
4923 895130 : void state_usage_file(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
4924 : {
4925 : (void)state;
4926 :
4927 895130 : disk->progress_file = file;
4928 895130 : }
4929 :
4930 899817 : void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
4931 : {
4932 899817 : uint64_t now = tick();
4933 899817 : uint64_t delta = now - state->tick_last;
4934 : unsigned i;
4935 :
4936 : /* increment the time spent in the data disks */
4937 942087 : for (i = 0; i < waiting_mac; ++i) {
4938 42270 : struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
4939 :
4940 42270 : if (!disk)
4941 1 : continue;
4942 :
4943 42269 : disk->tick += delta;
4944 : }
4945 899817 : state->tick_io += delta;
4946 :
4947 899817 : state->tick_last = now;
4948 899817 : }
4949 :
4950 761483 : void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
4951 : {
4952 761483 : uint64_t now = tick();
4953 761483 : uint64_t delta = now - state->tick_last;
4954 : unsigned i;
4955 :
4956 : /* increment the time spent in the parity disk */
4957 771430 : for (i = 0; i < waiting_mac; ++i)
4958 9947 : state->parity[waiting_map[i]].tick += delta;
4959 761483 : state->tick_io += delta;
4960 :
4961 761483 : state->tick_last = now;
4962 761483 : }
4963 :
4964 97 : void state_usage_print(struct snapraid_state* state)
4965 : {
4966 : /* set the latest data */
4967 97 : state_progress_latest(state);
4968 :
4969 97 : if (msg_level < MSG_PROGRESS)
4970 96 : return;
4971 :
4972 : /* print a graph for it */
4973 1 : state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
4974 : }
4975 :
4976 495 : void state_fscheck(struct snapraid_state* state, const char* ope)
4977 : {
4978 : tommy_node* i;
4979 :
4980 : /* check the file-system on all disks */
4981 3459 : for (i = state->disklist; i != 0; i = i->next) {
4982 2964 : struct snapraid_disk* disk = i->data;
4983 :
4984 2964 : if (fs_check(disk) != 0) {
4985 : /* LCOV_EXCL_START */
4986 : log_fatal("Internal inconsistency: File-system check for disk '%s' %s\n", disk->name, ope);
4987 : os_abort();
4988 : /* LCOV_EXCL_STOP */
4989 : }
4990 : }
4991 495 : }
4992 :
4993 1 : void generate_configuration(const char* path)
4994 : {
4995 : struct snapraid_state state;
4996 : struct snapraid_content* content;
4997 : char esc_buffer[ESC_MAX];
4998 : unsigned l, s;
4999 : tommy_node* j;
5000 :
5001 1 : state_init(&state);
5002 :
5003 : /* mark that we are without a configuration file */
5004 1 : state.no_conf = 1;
5005 :
5006 : /* create the dummy content entry */
5007 1 : content = content_alloc(path, -1);
5008 :
5009 : /* adds the content entry */
5010 1 : tommy_list_insert_tail(&state.contentlist, &content->node, content);
5011 :
5012 : /* read the content file */
5013 1 : state_read(&state);
5014 :
5015 : /* output a dummy configuration file */
5016 1 : printf("# Configuration file generated from %s\n", path);
5017 1 : printf("\n");
5018 1 : printf("# Use this blocksize\n");
5019 1 : printf("blocksize %u\n", state.block_size / KIBI);
5020 1 : printf("\n");
5021 1 : printf("# Use this hashsize\n");
5022 1 : printf("hashsize %u\n", BLOCK_HASH_SIZE);
5023 1 : printf("\n");
5024 7 : for (l = 0; l < state.level; ++l) {
5025 6 : printf("# Set the correct path for the %s files\n", lev_name(l));
5026 6 : printf("# You had %u of them:\n", state.parity[l].split_mac);
5027 30 : for (s = 0; s < state.parity[l].split_mac; ++s) {
5028 24 : printf("# %u:\n", s);
5029 24 : printf("# PATH:");
5030 24 : if (state.parity[l].split_map[s].path[0])
5031 24 : printf("%s", state.parity[l].split_map[s].path);
5032 : else
5033 0 : printf("?");
5034 24 : printf("\n");
5035 24 : printf("# SIZE:");
5036 24 : if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
5037 24 : printf("%" PRIu64, state.parity[l].split_map[s].size);
5038 : else
5039 0 : printf("?");
5040 24 : printf("\n");
5041 24 : printf("# UUID:");
5042 24 : if (state.parity[l].split_map[s].uuid[0])
5043 24 : printf("%s", state.parity[l].split_map[s].uuid);
5044 : else
5045 0 : printf("?");
5046 24 : printf("\n");
5047 24 : printf("#\n");
5048 : }
5049 6 : printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
5050 6 : printf("\n");
5051 : }
5052 1 : printf("# Add any other content file\n");
5053 1 : printf("content %s\n", path);
5054 1 : printf("\n");
5055 7 : for (j = state.maplist; j; j = j->next) {
5056 6 : struct snapraid_map* map = j->data;
5057 : struct snapraid_disk* disk;
5058 6 : printf("# Set the correct dir for disk '%s'\n", map->name);
5059 6 : if (map->uuid[0])
5060 6 : printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
5061 6 : disk = find_disk_by_name(&state, map->name);
5062 6 : if (disk && disk->filelist) {
5063 6 : struct snapraid_file* file = disk->filelist->data;
5064 6 : if (file) {
5065 6 : printf("# and containing: %s\n", fmt_poll(disk, file->sub, esc_buffer));
5066 : }
5067 : }
5068 6 : printf("data %s ENTER_HERE_THE_DIR\n", map->name);
5069 6 : printf("\n");
5070 : }
5071 :
5072 1 : state_done(&state);
5073 1 : }
5074 :
5075 :
5076 : /**
5077 : * Flush and sync the parity files.
5078 : */
5079 84 : int state_flush(struct snapraid_state* state, struct snapraid_io* io, struct snapraid_parity_handle* parity_handle, block_off_t blockcur)
5080 : {
5081 : unsigned l;
5082 :
5083 84 : if (io)
5084 84 : io_flush(io);
5085 :
5086 : /* flush all parity handles to ensure data is written to disk */
5087 553 : for (l = 0; l < state->level; ++l) {
5088 469 : int ret = parity_sync(&parity_handle[l]);
5089 469 : if (ret == -1) {
5090 : /* LCOV_EXCL_START */
5091 : log_tag("parity_error:%u:%s: Sync error\n", blockcur, lev_config_name(l));
5092 : log_fatal("DANGER! Unexpected sync error in %s disk.\n", lev_name(l));
5093 : log_fatal("Ensure that disk '%s' is sane.\n", lev_config_name(l));
5094 : return -1;
5095 : /* LCOV_EXCL_STOP */
5096 : }
5097 : }
5098 :
5099 84 : return 0;
5100 : }
5101 :
|
