Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "elem.h"
22 : #include "state.h"
23 : #include "parity.h"
24 : #include "handle.h"
25 : #include "io.h"
26 : #include "raid/raid.h"
27 :
28 : /****************************************************************************/
29 : /* hash */
30 :
31 7 : static int state_hash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, int* skip_sync)
32 : {
33 : struct snapraid_handle* handle;
34 : unsigned diskmax;
35 : block_off_t i;
36 : unsigned j;
37 : void* buffer;
38 : void* buffer_alloc;
39 : data_off_t countsize;
40 : block_off_t countpos;
41 : block_off_t countmax;
42 : int ret;
43 : unsigned error;
44 : unsigned silent_error;
45 : unsigned io_error;
46 : char esc_buffer[ESC_MAX];
47 :
48 : /* maps the disks to handles */
49 7 : handle = handle_mapping(state, &diskmax);
50 :
51 : /* buffer for reading */
52 7 : buffer = malloc_nofail_direct(state->block_size, &buffer_alloc);
53 7 : if (!state->opt.skip_self)
54 0 : mtest_vector(1, state->block_size, &buffer);
55 :
56 7 : error = 0;
57 7 : silent_error = 0;
58 7 : io_error = 0;
59 :
60 : /* first count the number of blocks to process */
61 7 : countmax = 0;
62 49 : for (j = 0; j < diskmax; ++j) {
63 42 : struct snapraid_disk* disk = handle[j].disk;
64 :
65 : /* if no disk, nothing to check */
66 42 : if (!disk)
67 0 : continue;
68 :
69 236670 : for (i = blockstart; i < blockmax; ++i) {
70 : struct snapraid_block* block;
71 : unsigned block_state;
72 :
73 236628 : block = fs_par2block_find(disk, i);
74 :
75 : /* get the state of the block */
76 236628 : block_state = block_state_get(block);
77 :
78 : /* process REP and CHG blocks */
79 236628 : if (block_state != BLOCK_STATE_REP && block_state != BLOCK_STATE_CHG)
80 223278 : continue;
81 :
82 13350 : ++countmax;
83 : }
84 : }
85 :
86 : /* drop until now */
87 7 : state_usage_waste(state);
88 :
89 7 : countsize = 0;
90 7 : countpos = 0;
91 7 : if (!state_progress_begin(state, blockstart, blockmax, countmax))
92 0 : goto end;
93 :
94 49 : for (j = 0; j < diskmax; ++j) {
95 42 : struct snapraid_disk* disk = handle[j].disk;
96 :
97 : /* if no disk, nothing to check */
98 42 : if (!disk)
99 0 : continue;
100 :
101 236670 : for (i = blockstart; i < blockmax; ++i) {
102 : snapraid_info info;
103 : int rehash;
104 : struct snapraid_block* block;
105 : int read_size;
106 : unsigned char hash[HASH_MAX];
107 : unsigned block_state;
108 : struct snapraid_file* file;
109 : block_off_t file_pos;
110 :
111 236628 : block = fs_par2block_find(disk, i);
112 :
113 : /* get the state of the block */
114 236628 : block_state = block_state_get(block);
115 :
116 : /* process REP and CHG blocks */
117 236628 : if (block_state != BLOCK_STATE_REP && block_state != BLOCK_STATE_CHG)
118 446566 : continue;
119 :
120 : /* get the file of this block */
121 13350 : file = fs_par2file_get(disk, i, &file_pos);
122 :
123 : /* get block specific info */
124 13350 : info = info_get(&state->infoarr, i);
125 :
126 : /* if we have to use the old hash */
127 13350 : rehash = info_get_rehash(info);
128 :
129 : /* until now is misc */
130 13350 : state_usage_misc(state);
131 :
132 : /* if the file is different than the current one, close it */
133 13350 : if (handle[j].file != 0 && handle[j].file != file) {
134 : /* keep a pointer at the file we are going to close for error reporting */
135 5366 : struct snapraid_file* report = handle[j].file;
136 5366 : ret = handle_close(&handle[j]);
137 5366 : if (ret == -1) {
138 : /* LCOV_EXCL_START */
139 : /* This one is really an unexpected error, because we are only reading */
140 : /* and closing a descriptor should never fail */
141 : if (errno == EIO) {
142 : log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
143 : log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
144 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
145 : log_fatal("Stopping at block %u\n", i);
146 : ++io_error;
147 : goto bail;
148 : }
149 :
150 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
151 : log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
152 : log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
153 : log_fatal("Stopping at block %u\n", i);
154 : ++error;
155 : goto bail;
156 : /* LCOV_EXCL_STOP */
157 : }
158 : }
159 :
160 13350 : ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
161 13350 : if (ret == -1) {
162 6 : if (errno == EIO) {
163 : /* LCOV_EXCL_START */
164 : log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
165 : log_fatal("DANGER! Unexpected input/output open error in a data disk, it isn't possible to sync.\n");
166 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
167 : log_fatal("Stopping at block %u\n", i);
168 : ++io_error;
169 : goto bail;
170 : /* LCOV_EXCL_STOP */
171 : }
172 :
173 6 : if (errno == ENOENT) {
174 2 : log_tag("error:%u:%s:%s: Open ENOENT error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
175 2 : log_error("Missing file '%s'.\n", handle[j].path);
176 2 : log_error("WARNING! You cannot modify data disk during a sync.\n");
177 2 : log_error("Rerun the sync command when finished.\n");
178 2 : ++error;
179 : /* if the file is missing, it means that it was removed during sync */
180 : /* this isn't a serious error, so we skip this block, and continue with others */
181 2 : continue;
182 : }
183 :
184 4 : if (errno == EACCES) {
185 4 : log_tag("error:%u:%s:%s: Open EACCES error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
186 4 : log_error("No access at file '%s'.\n", handle[j].path);
187 4 : log_error("WARNING! Please fix the access permission in the data disk.\n");
188 4 : log_error("Rerun the sync command when finished.\n");
189 4 : ++error;
190 : /* this isn't a serious error, so we skip this block, and continue with others */
191 4 : continue;
192 : }
193 :
194 : /* LCOV_EXCL_START */
195 : log_tag("error:%u:%s:%s: Open error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
196 : log_fatal("WARNING! Unexpected open error in a data disk, it isn't possible to sync.\n");
197 : log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
198 : log_fatal("Stopping to allow recovery. Try with 'snapraid check -f /%s'\n", fmt_poll(disk, file->sub, esc_buffer));
199 : ++error;
200 : goto bail;
201 : /* LCOV_EXCL_STOP */
202 : }
203 :
204 : /* check if the file is changed */
205 13344 : if (handle[j].st.st_size != file->size
206 13343 : || handle[j].st.st_mtime != file->mtime_sec
207 13342 : || STAT_NSEC(&handle[j].st) != file->mtime_nsec
208 13342 : || handle[j].st.st_ino != file->inode
209 : ) {
210 3 : log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc_tag(file->sub, esc_buffer));
211 3 : if (handle[j].st.st_size != file->size) {
212 1 : log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, (uint64_t)handle[j].st.st_size);
213 2 : } else if (handle[j].st.st_mtime != file->mtime_sec
214 1 : || STAT_NSEC(&handle[j].st) != file->mtime_nsec) {
215 1 : log_error("Unexpected time change at file '%s' from %" PRIu64 ".%d to %" PRIu64 ".%d.\n", handle[j].path, file->mtime_sec, file->mtime_nsec, (uint64_t)handle[j].st.st_mtime, STAT_NSEC(&handle[j].st));
216 : } else {
217 1 : log_error("Unexpected inode change from %" PRIu64 " to %" PRIu64 " at file '%s'.\n", file->inode, (uint64_t)handle[j].st.st_ino, handle[j].path);
218 : }
219 3 : log_error("WARNING! You cannot modify files during a sync.\n");
220 3 : log_error("Rerun the sync command when finished.\n");
221 3 : ++error;
222 : /* if the file is changed, it means that it was modified during sync */
223 : /* this isn't a serious error, so we skip this block, and continue with others */
224 3 : continue;
225 : }
226 :
227 13341 : read_size = handle_read(&handle[j], file_pos, buffer, state->block_size, log_fatal, 0);
228 13341 : if (read_size == -1) {
229 : /* LCOV_EXCL_START */
230 : if (errno == EIO) {
231 : log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), file_pos, strerror(errno));
232 : log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
233 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path);
234 : log_fatal("Stopping at block %u\n", i);
235 : ++io_error;
236 : goto bail;
237 : }
238 :
239 : log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), file_pos, strerror(errno));
240 : log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
241 : log_fatal("Ensure that file '%s' can be read.\n", handle[j].path);
242 : log_fatal("Stopping to allow recovery. Try with 'snapraid check -f /%s'\n", fmt_poll(disk, file->sub, esc_buffer));
243 : ++error;
244 : goto bail;
245 : /* LCOV_EXCL_STOP */
246 : }
247 :
248 : /* until now is disk */
249 13341 : state_usage_disk(state, handle, &j, 1);
250 :
251 13341 : countsize += read_size;
252 :
253 : /* now compute the hash */
254 13341 : if (rehash) {
255 0 : memhash(state->prevhash, state->prevhashseed, hash, buffer, read_size);
256 : } else {
257 13341 : memhash(state->hash, state->hashseed, hash, buffer, read_size);
258 : }
259 :
260 : /* until now is hash */
261 13341 : state_usage_hash(state);
262 :
263 13341 : if (block_state == BLOCK_STATE_REP) {
264 : /* compare the hash */
265 13339 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
266 1 : log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc_tag(file->sub, esc_buffer));
267 1 : log_error("Data change at file '%s' at position '%u'\n", handle[j].path, file_pos);
268 1 : log_error("WARNING! Unexpected data modification of a file without parity!\n");
269 :
270 1 : if (file_flag_has(file, FILE_IS_COPY)) {
271 1 : log_error("This file was detected as a copy of another file with the same name, size,\n");
272 1 : log_error("and timestamp, but the file data isn't matching the assumed copy.\n");
273 1 : log_error("If this is a false positive, and the files are expected to be different,\n");
274 1 : log_error("you can 'sync' anyway using 'snapraid --force-nocopy sync'\n");
275 : } else {
276 0 : log_error("Try removing the file from the array and rerun the 'sync' command!\n");
277 : }
278 :
279 : /* block sync to allow a recovery before overwriting */
280 : /* the parity needed to make such recovery */
281 1 : *skip_sync = 1; /* avoid to run the next sync */
282 :
283 1 : ++silent_error;
284 1 : continue;
285 : }
286 : } else {
287 : /* the only other case is BLOCK_STATE_CHG */
288 2 : assert(block_state == BLOCK_STATE_CHG);
289 :
290 : /* copy the hash in the block */
291 2 : memcpy(block->hash, hash, BLOCK_HASH_SIZE);
292 :
293 : /* and mark the block as hashed */
294 2 : block_state_set(block, BLOCK_STATE_REP);
295 :
296 : /* mark the state as needing write */
297 2 : state->need_write = 1;
298 : }
299 :
300 : /* count the number of processed block */
301 13340 : ++countpos;
302 :
303 : /* progress */
304 13340 : if (state_progress(state, 0, i, countpos, countmax, countsize)) {
305 : /* LCOV_EXCL_START */
306 : *skip_sync = 1; /* avoid to run the next sync */
307 : break;
308 : /* LCOV_EXCL_STOP */
309 : }
310 : }
311 :
312 : /* close the last file in the disk */
313 42 : if (handle[j].file != 0) {
314 : /* keep a pointer at the file we are going to close for error reporting */
315 7 : struct snapraid_file* report = handle[j].file;
316 7 : ret = handle_close(&handle[j]);
317 7 : if (ret == -1) {
318 : /* LCOV_EXCL_START */
319 : /* This one is really an unexpected error, because we are only reading */
320 : /* and closing a descriptor should never fail */
321 : if (errno == EIO) {
322 : log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
323 : log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
324 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
325 : log_fatal("Stopping at block %u\n", blockmax);
326 : ++io_error;
327 : goto bail;
328 : }
329 :
330 : log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
331 : log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
332 : log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
333 : log_fatal("Stopping at block %u\n", blockmax);
334 : ++error;
335 : goto bail;
336 : /* LCOV_EXCL_STOP */
337 : }
338 : }
339 : }
340 :
341 : end:
342 7 : state_progress_end(state, countpos, countmax, countsize);
343 :
344 : /* note that at this point no io_error is possible */
345 : /* because at the first one we bail out */
346 7 : assert(io_error == 0);
347 :
348 7 : if (error || io_error || silent_error) {
349 6 : msg_status("\n");
350 6 : msg_status("%8u file errors\n", error);
351 6 : msg_status("%8u io errors\n", io_error);
352 6 : msg_status("%8u data errors\n", silent_error);
353 : } else {
354 : /* print the result only if processed something */
355 1 : if (countpos != 0)
356 1 : msg_status("Everything OK\n");
357 : }
358 :
359 7 : if (error)
360 5 : log_fatal("WARNING! Unexpected file errors!\n");
361 :
362 7 : log_tag("hash_summary:error_file:%u\n", error);
363 :
364 : /* proceed without bailing out */
365 7 : goto finish;
366 :
367 : bail:
368 : /* on bail, don't run the next sync */
369 0 : *skip_sync = 1;
370 :
371 : /* close files left open */
372 0 : for (j = 0; j < diskmax; ++j) {
373 0 : struct snapraid_file* file = handle[j].file;
374 0 : struct snapraid_disk* disk = handle[j].disk;
375 0 : ret = handle_close(&handle[j]);
376 0 : if (ret == -1) {
377 0 : log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
378 0 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
379 0 : ++error;
380 : /* continue, as we are already exiting */
381 : }
382 : }
383 :
384 : finish:
385 7 : free(handle);
386 7 : free(buffer_alloc);
387 :
388 7 : if (error + io_error + silent_error != 0)
389 6 : return -1;
390 1 : return 0;
391 : }
392 :
393 : /****************************************************************************/
394 : /* sync */
395 :
396 : /**
397 : * Sync plan to use.
398 : */
399 : struct snapraid_plan {
400 : unsigned handle_max;
401 : struct snapraid_handle* handle_map;
402 : int force_full;
403 : };
404 :
405 : /**
406 : * A block that failed the hash check, or that was deleted.
407 : */
408 : struct failed_struct {
409 : unsigned index; /**< Index of the failed block. */
410 : unsigned size; /**< Size of the block. */
411 :
412 : struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */
413 : };
414 :
415 : /**
416 : * Comparison function for sorting by index.
417 : */
418 853 : int failed_compare_by_index(const void* void_a, const void* void_b)
419 : {
420 853 : const struct failed_struct* a = void_a;
421 853 : const struct failed_struct* b = void_b;
422 :
423 853 : if (a->index < b->index)
424 853 : return -1;
425 0 : if (a->index > b->index)
426 0 : return 1;
427 0 : return 0;
428 : }
429 :
430 : /**
431 : * Buffer for storing the new hashes.
432 : */
433 : struct snapraid_rehash {
434 : unsigned char hash[HASH_MAX];
435 : struct snapraid_block* block;
436 : };
437 :
438 : /**
439 : * Check if we have to process the specified block index ::i.
440 : */
441 820170 : static int block_is_enabled(void* void_plan, block_off_t i)
442 : {
443 820170 : struct snapraid_plan* plan = void_plan;
444 : unsigned j;
445 : int one_invalid;
446 : int one_valid;
447 :
448 : /* for each disk */
449 820170 : one_invalid = 0;
450 820170 : one_valid = 0;
451 5741190 : for (j = 0; j < plan->handle_max; ++j) {
452 : struct snapraid_block* block;
453 4921020 : struct snapraid_disk* disk = plan->handle_map[j].disk;
454 :
455 : /* if no disk, nothing to check */
456 4921020 : if (!disk)
457 9374 : continue;
458 :
459 4911646 : block = fs_par2block_find(disk, i);
460 :
461 4911646 : if (block_has_file(block))
462 4688742 : one_valid = 1;
463 :
464 4911646 : if (block_has_invalid_parity(block) || plan->force_full)
465 753212 : one_invalid = 1;
466 : }
467 :
468 : /* if none valid or none invalid, we don't need to update */
469 820170 : if (!one_invalid || !one_valid)
470 590658 : return 0;
471 :
472 229512 : return 1;
473 : }
474 :
475 687996 : static void sync_data_reader(struct snapraid_worker* worker, struct snapraid_task* task)
476 : {
477 687996 : struct snapraid_io* io = worker->io;
478 687996 : struct snapraid_state* state = io->state;
479 687996 : struct snapraid_handle* handle = worker->handle;
480 687996 : struct snapraid_disk* disk = handle->disk;
481 687996 : block_off_t blockcur = task->position;
482 687996 : unsigned char* buffer = task->buffer;
483 : int ret;
484 : char esc_buffer[ESC_MAX];
485 :
486 : /* if the disk position is not used */
487 687996 : if (!disk) {
488 : /* use an empty block */
489 0 : memset(buffer, 0, state->block_size);
490 0 : task->state = TASK_STATE_DONE;
491 91851 : return;
492 : }
493 :
494 : /* get the block */
495 688021 : task->block = fs_par2block_find(disk, blockcur);
496 :
497 : /* if the block has no file, meaning that it's EMPTY or DELETED, */
498 : /* it doesn't participate in the new parity computation */
499 686555 : if (!block_has_file(task->block)) {
500 : /* use an empty block */
501 91883 : memset(buffer, 0, state->block_size);
502 91883 : task->state = TASK_STATE_DONE;
503 91883 : return;
504 : }
505 :
506 : /* get the file of this block */
507 594348 : task->file = fs_par2file_get(disk, blockcur, &task->file_pos);
508 :
509 : /* if the file is different than the current one, close it */
510 595908 : if (handle->file != 0 && handle->file != task->file) {
511 : /* keep a pointer at the file we are going to close for error reporting */
512 246073 : struct snapraid_file* report = handle->file;
513 246073 : ret = handle_close(handle);
514 243940 : if (ret == -1) {
515 : /* LCOV_EXCL_START */
516 : /* This one is really an unexpected error, because we are only reading */
517 : /* and closing a descriptor should never fail */
518 : if (errno == EIO) {
519 : log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockcur, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
520 : log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
521 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle->path);
522 : log_fatal("Stopping at block %u\n", blockcur);
523 : task->state = TASK_STATE_IOERROR;
524 : return;
525 : }
526 :
527 : log_tag("error:%u:%s:%s: Close error. %s\n", blockcur, disk->name, esc_tag(report->sub, esc_buffer), strerror(errno));
528 : log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
529 : log_fatal("Ensure that file '%s' can be accessed.\n", handle->path);
530 : log_fatal("Stopping at block %u\n", blockcur);
531 : task->state = TASK_STATE_ERROR;
532 : return;
533 : /* LCOV_EXCL_STOP */
534 : }
535 : }
536 :
537 593775 : ret = handle_open(handle, task->file, state->file_mode, log_error, 0);
538 588780 : if (ret == -1) {
539 12 : if (errno == EIO) {
540 : /* LCOV_EXCL_START */
541 : log_tag("error:%u:%s:%s: Open EIO error. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
542 : log_fatal("DANGER! Unexpected input/output open error in a data disk, it isn't possible to sync.\n");
543 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle->path);
544 : log_fatal("Stopping at block %u\n", blockcur);
545 : task->state = TASK_STATE_IOERROR;
546 : return;
547 : /* LCOV_EXCL_STOP */
548 : }
549 :
550 12 : if (errno == ENOENT) {
551 4 : log_tag("error:%u:%s:%s: Open ENOENT error. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
552 4 : log_error("Missing file '%s'.\n", handle->path);
553 4 : log_error("WARNING! You cannot modify data disk during a sync.\n");
554 4 : log_error("Rerun the sync command when finished.\n");
555 : /* if the file is missing, it means that it was removed during sync */
556 : /* this isn't a serious error, so we skip this block, and continue with others */
557 4 : task->state = TASK_STATE_ERROR_CONTINUE;
558 4 : return;
559 : }
560 :
561 8 : if (errno == EACCES) {
562 8 : log_tag("error:%u:%s:%s: Open EACCES error. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
563 8 : log_error("No access at file '%s'.\n", handle->path);
564 8 : log_error("WARNING! Please fix the access permission in the data disk.\n");
565 8 : log_error("Rerun the sync command when finished.\n");
566 : /* this isn't a serious error, so we skip this block, and continue with others */
567 8 : task->state = TASK_STATE_ERROR_CONTINUE;
568 8 : return;
569 : }
570 :
571 : /* LCOV_EXCL_START */
572 : log_tag("error:%u:%s:%s: Open error. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), strerror(errno));
573 : log_fatal("WARNING! Unexpected open error in a data disk, it isn't possible to sync.\n");
574 : log_fatal("Ensure that file '%s' can be accessed.\n", handle->path);
575 : log_fatal("Stopping to allow recovery. Try with 'snapraid check -f /%s'\n", fmt_poll(disk, task->file->sub, esc_buffer));
576 : task->state = TASK_STATE_ERROR;
577 : return;
578 : /* LCOV_EXCL_STOP */
579 : }
580 :
581 : /* check if the file is changed */
582 588768 : if (handle->st.st_size != task->file->size
583 591040 : || handle->st.st_mtime != task->file->mtime_sec
584 593884 : || STAT_NSEC(&handle->st) != task->file->mtime_nsec
585 593616 : || handle->st.st_ino != task->file->inode
586 : ) {
587 0 : log_tag("error:%u:%s:%s: Unexpected attribute change\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer));
588 6 : if (handle->st.st_size != task->file->size) {
589 2 : log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle->path, task->file->size, (uint64_t)handle->st.st_size);
590 4 : } else if (handle->st.st_mtime != task->file->mtime_sec
591 2 : || STAT_NSEC(&handle->st) != task->file->mtime_nsec) {
592 2 : log_error("Unexpected time change at file '%s' from %" PRIu64 ".%d to %" PRIu64 ".%d.\n", handle->path, task->file->mtime_sec, task->file->mtime_nsec, (uint64_t)handle->st.st_mtime, STAT_NSEC(&handle->st));
593 : } else {
594 2 : log_error("Unexpected inode change from %" PRIu64 " to %" PRIu64 " at file '%s'.\n", task->file->inode, (uint64_t)handle->st.st_ino, handle->path);
595 : }
596 6 : log_error("WARNING! You cannot modify files during a sync.\n");
597 6 : log_error("Rerun the sync command when finished.\n");
598 : /* if the file is changed, it means that it was modified during sync */
599 : /* this isn't a serious error, so we skip this block, and continue with others */
600 6 : task->state = TASK_STATE_ERROR_CONTINUE;
601 6 : return;
602 : }
603 :
604 595293 : task->read_size = handle_read(handle, task->file_pos, buffer, state->block_size, log_error, 0);
605 592356 : if (task->read_size == -1) {
606 : /* LCOV_EXCL_START */
607 : if (errno == EIO) {
608 : log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), task->file_pos, strerror(errno));
609 : log_error("Input/Output error in file '%s' at position '%u'\n", handle->path, task->file_pos);
610 : task->state = TASK_STATE_IOERROR_CONTINUE;
611 : return;
612 : }
613 :
614 : log_tag("error:%u:%s:%s: Read error at position %u. %s\n", blockcur, disk->name, esc_tag(task->file->sub, esc_buffer), task->file_pos, strerror(errno));
615 : log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
616 : log_fatal("Ensure that file '%s' can be read.\n", handle->path);
617 : log_fatal("Stopping to allow recovery. Try with 'snapraid check -f /%s'\n", fmt_poll(disk, task->file->sub, esc_buffer));
618 : task->state = TASK_STATE_ERROR;
619 : return;
620 : /* LCOV_EXCL_STOP */
621 : }
622 :
623 : /* store the path of the opened file */
624 592356 : pathcpy(task->path, sizeof(task->path), handle->path);
625 :
626 588506 : task->state = TASK_STATE_DONE;
627 : }
628 :
629 543417 : static void sync_parity_writer(struct snapraid_worker* worker, struct snapraid_task* task)
630 : {
631 543417 : struct snapraid_io* io = worker->io;
632 543417 : struct snapraid_state* state = io->state;
633 543417 : struct snapraid_parity_handle* parity_handle = worker->parity_handle;
634 543417 : unsigned level = parity_handle->level;
635 543417 : block_off_t blockcur = task->position;
636 543417 : unsigned char* buffer = task->buffer;
637 : int ret;
638 :
639 : /* write parity */
640 543417 : ret = parity_write(parity_handle, blockcur, buffer, state->block_size);
641 541047 : if (ret == -1) {
642 : /* LCOV_EXCL_START */
643 : if (errno == EIO) {
644 : log_tag("parity_error:%u:%s: Write EIO error. %s\n", blockcur, lev_config_name(level), strerror(errno));
645 : log_error("Input/Output error in parity '%s' at position '%u'\n", lev_config_name(level), blockcur);
646 : task->state = TASK_STATE_IOERROR_CONTINUE;
647 : return;
648 : }
649 :
650 : log_tag("parity_error:%u:%s: Write error. %s\n", blockcur, lev_config_name(level), strerror(errno));
651 : log_fatal("WARNING! Unexpected write error in the %s disk, it isn't possible to sync.\n", lev_name(level));
652 : log_fatal("Ensure that disk '%s' has some free space available.\n", lev_config_name(level));
653 : log_fatal("Stopping at block %u\n", blockcur);
654 : task->state = TASK_STATE_ERROR;
655 : return;
656 : /* LCOV_EXCL_STOP */
657 : }
658 :
659 541047 : task->state = TASK_STATE_DONE;
660 : }
661 :
662 83 : static int state_sync_process(struct snapraid_state* state, struct snapraid_parity_handle* parity_handle, block_off_t blockstart, block_off_t blockmax)
663 : {
664 : struct snapraid_io io;
665 : struct snapraid_plan plan;
666 : struct snapraid_handle* handle;
667 : void* rehandle_alloc;
668 : struct snapraid_rehash* rehandle;
669 : unsigned diskmax;
670 : block_off_t blockcur;
671 : unsigned j;
672 : void* zero_alloc;
673 : void** zero;
674 : void* copy_alloc;
675 : void** copy;
676 : unsigned buffermax;
677 : data_off_t countsize;
678 : block_off_t countpos;
679 : block_off_t countmax;
680 : block_off_t autosavedone;
681 : block_off_t autosavelimit;
682 : block_off_t autosavemissing;
683 : int ret;
684 : unsigned error;
685 : unsigned silent_error;
686 : unsigned io_error;
687 : time_t now;
688 : struct failed_struct* failed;
689 : int* failed_map;
690 : unsigned l;
691 : unsigned* waiting_map;
692 : unsigned waiting_mac;
693 : char esc_buffer[ESC_MAX];
694 :
695 : /* the sync process assumes that all the hashes are correct */
696 : /* including the ones from CHG and DELETED blocks */
697 83 : assert(state->clear_past_hash != 0);
698 :
699 : /* get the present time */
700 83 : now = time(0);
701 :
702 : /* maps the disks to handles */
703 83 : handle = handle_mapping(state, &diskmax);
704 :
705 : /* rehash buffers */
706 83 : rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
707 :
708 : /* we need 1 * data + 1 * parity */
709 83 : buffermax = diskmax + state->level;
710 :
711 : /* initialize the io threads */
712 83 : io_init(&io, state, state->opt.io_cache, buffermax, sync_data_reader, handle, diskmax, 0, sync_parity_writer, parity_handle, state->level);
713 :
714 : /* allocate the copy buffer */
715 83 : copy = malloc_nofail_vector_align(diskmax, diskmax, state->block_size, ©_alloc);
716 :
717 : /* allocate and fill the zero buffer */
718 83 : zero = malloc_nofail_align(state->block_size, &zero_alloc);
719 83 : memset(zero, 0, state->block_size);
720 83 : raid_zero(zero);
721 :
722 83 : failed = malloc_nofail(diskmax * sizeof(struct failed_struct));
723 83 : failed_map = malloc_nofail(diskmax * sizeof(unsigned));
724 :
725 : /* possibly waiting disks */
726 83 : waiting_mac = diskmax > RAID_PARITY_MAX ? diskmax : RAID_PARITY_MAX;
727 83 : waiting_map = malloc_nofail(waiting_mac * sizeof(unsigned));
728 :
729 83 : error = 0;
730 83 : silent_error = 0;
731 83 : io_error = 0;
732 :
733 : /* first count the number of blocks to process */
734 83 : countmax = 0;
735 83 : plan.handle_max = diskmax;
736 83 : plan.handle_map = handle;
737 83 : plan.force_full = state->opt.force_full;
738 410168 : for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
739 410085 : if (!block_is_enabled(&plan, blockcur))
740 295329 : continue;
741 114756 : ++countmax;
742 : }
743 :
744 : /* compute the autosave size for all disk, even if not read */
745 : /* this makes sense because the speed should be almost the same */
746 : /* if the disks are read in parallel */
747 83 : autosavelimit = state->autosave / (diskmax * state->block_size);
748 83 : autosavemissing = countmax; /* blocks to do */
749 83 : autosavedone = 0; /* blocks done */
750 :
751 : /* drop until now */
752 83 : state_usage_waste(state);
753 :
754 83 : countsize = 0;
755 83 : countpos = 0;
756 :
757 : /* start all the worker threads */
758 83 : io_start(&io, blockstart, blockmax, &block_is_enabled, &plan);
759 :
760 83 : if (!state_progress_begin(state, blockstart, blockmax, countmax))
761 0 : goto end;
762 :
763 : while (1) {
764 : unsigned failed_count;
765 : int error_on_this_block;
766 : int silent_error_on_this_block;
767 : int io_error_on_this_block;
768 : int fixed_error_on_this_block;
769 : int parity_needs_to_be_updated;
770 : int parity_going_to_be_updated;
771 : snapraid_info info;
772 : int rehash;
773 : void** buffer;
774 : int writer_error[IO_WRITER_ERROR_MAX];
775 :
776 : /* go to the next block */
777 114839 : blockcur = io_read_next(&io, &buffer);
778 114839 : if (blockcur >= blockmax)
779 166 : break;
780 :
781 : /* until now is scheduling */
782 114756 : state_usage_sched(state);
783 :
784 : /* one more block processed for autosave */
785 114756 : ++autosavedone;
786 114756 : --autosavemissing;
787 :
788 : /* by default process the block, and skip it if something goes wrong */
789 114756 : error_on_this_block = 0;
790 114756 : silent_error_on_this_block = 0;
791 114756 : io_error_on_this_block = 0;
792 114756 : fixed_error_on_this_block = 0;
793 :
794 : /* keep track of the number of failed blocks */
795 114756 : failed_count = 0;
796 :
797 : /* get block specific info */
798 114756 : info = info_get(&state->infoarr, blockcur);
799 :
800 : /* if we have to use the old hash */
801 114756 : rehash = info_get_rehash(info);
802 :
803 : /* if the parity requires to be updated */
804 : /* It could happens that all the blocks are EMPTY/BLK and CHG but with the hash */
805 : /* still matching because the specific CHG block was not modified. */
806 : /* In such case, we can avoid to update parity, because it would be the same as before */
807 : /* Note that CHG/DELETED blocks already present in the content file loaded */
808 : /* have the hash cleared (::clear_past_hash flag), and then they won't never match the hash. */
809 : /* We are treating only CHG blocks created at runtime. */
810 114756 : parity_needs_to_be_updated = state->opt.force_full || state->opt.force_parity_update;
811 :
812 : /* if the parity is going to be updated */
813 114756 : parity_going_to_be_updated = 0;
814 :
815 : /* if the block is marked as bad, we force the parity update */
816 : /* because the bad block may be the result of a wrong parity */
817 114756 : if (info_get_bad(info))
818 0 : parity_needs_to_be_updated = 1;
819 :
820 : /* for each disk, process the block */
821 803292 : for (j = 0; j < diskmax; ++j) {
822 : struct snapraid_task* task;
823 : int read_size;
824 : unsigned char hash[HASH_MAX];
825 : struct snapraid_block* block;
826 : unsigned block_state;
827 : struct snapraid_disk* disk;
828 : struct snapraid_file* file;
829 : block_off_t file_pos;
830 : unsigned diskcur;
831 :
832 : /* until now is misc */
833 688536 : state_usage_misc(state);
834 :
835 688536 : task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
836 :
837 : /* until now is disk */
838 688536 : state_usage_disk(state, handle, waiting_map, waiting_mac);
839 :
840 : /* get the results */
841 688536 : disk = task->disk;
842 688536 : block = task->block;
843 688536 : file = task->file;
844 688536 : file_pos = task->file_pos;
845 688536 : read_size = task->read_size;
846 :
847 : /* by default no rehash in case of "continue" */
848 688536 : rehandle[diskcur].block = 0;
849 :
850 : /* if the disk position is not used */
851 688536 : if (!disk)
852 93057 : continue;
853 :
854 : /* get the state of the block */
855 688536 : block_state = block_state_get(block);
856 :
857 : /* if the block has invalid parity, */
858 : /* we have to take care of it in case of recover */
859 688536 : if (block_has_invalid_parity(block)) {
860 : /* store it in the failed set, because */
861 : /* the parity may be still computed with the previous content */
862 182811 : failed[failed_count].index = diskcur;
863 182811 : failed[failed_count].size = state->block_size;
864 182811 : failed[failed_count].block = block;
865 182811 : ++failed_count;
866 :
867 : /* if the block has invalid parity, we have to update the parity */
868 : /* to include this block change */
869 : /* This also apply to CHG blocks, but we are going to handle */
870 : /* later this case to do the updates only if really needed */
871 182811 : if (block_state != BLOCK_STATE_CHG)
872 88225 : parity_needs_to_be_updated = 1;
873 :
874 : /* note that DELETE blocks are skipped in the next check */
875 : /* and we have to store them in the failed blocks */
876 : /* before skipping */
877 :
878 : /* follow */
879 : }
880 :
881 : /* if the block is not used */
882 688536 : if (!block_has_file(block))
883 92185 : continue;
884 :
885 : /* handle error conditions */
886 596351 : if (task->state == TASK_STATE_IOERROR) {
887 : /* LCOV_EXCL_START */
888 : ++io_error;
889 : goto bail;
890 : /* LCOV_EXCL_STOP */
891 : }
892 596351 : if (task->state == TASK_STATE_ERROR) {
893 : /* LCOV_EXCL_START */
894 : ++error;
895 : goto bail;
896 : /* LCOV_EXCL_STOP */
897 : }
898 596351 : if (task->state == TASK_STATE_ERROR_CONTINUE) {
899 18 : ++error;
900 18 : error_on_this_block = 1;
901 18 : continue;
902 : }
903 596333 : if (task->state == TASK_STATE_IOERROR_CONTINUE) {
904 0 : ++io_error;
905 0 : if (io_error >= state->opt.io_error_limit) {
906 : /* LCOV_EXCL_START */
907 : log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
908 : log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, task->path);
909 : log_fatal("Stopping at block %u\n", blockcur);
910 : goto bail;
911 : /* LCOV_EXCL_STOP */
912 : }
913 :
914 : /* otherwise continue */
915 0 : io_error_on_this_block = 1;
916 0 : continue;
917 : }
918 596333 : if (task->state != TASK_STATE_DONE) {
919 : /* LCOV_EXCL_START */
920 : log_fatal("Internal inconsistency in task state\n");
921 : os_abort();
922 : /* LCOV_EXCL_STOP */
923 : }
924 :
925 596333 : countsize += read_size;
926 :
927 : /* now compute the hash */
928 596333 : if (rehash) {
929 27243 : memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
930 :
931 : /* compute the new hash, and store it */
932 27243 : rehandle[diskcur].block = block;
933 27243 : memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
934 : } else {
935 569090 : memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
936 : }
937 :
938 : /* until now is hash */
939 596333 : state_usage_hash(state);
940 :
941 596333 : if (block_has_updated_hash(block)) {
942 : /* compare the hash */
943 501765 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
944 : /* if the file has invalid parity, it's a REP changed during the sync */
945 854 : if (block_has_invalid_parity(block)) {
946 1 : log_tag("error:%u:%s:%s: Unexpected data change\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer));
947 1 : log_error("Data change at file '%s' at position '%u'\n", task->path, file_pos);
948 1 : log_error("WARNING! Unexpected data modification of a file without parity!\n");
949 :
950 1 : if (file_flag_has(file, FILE_IS_COPY)) {
951 0 : log_error("This file was detected as a copy of another file with the same name, size,\n");
952 0 : log_error("and timestamp, but the file data isn't matching the assumed copy.\n");
953 0 : log_error("If this is a false positive, and the files are expected to be different,\n");
954 0 : log_error("you can 'sync' anyway using 'snapraid --force-nocopy sync'\n");
955 : } else {
956 1 : log_error("Try removing the file from the array and rerun the 'sync' command!\n");
957 : }
958 :
959 1 : ++error;
960 :
961 : /* if the file is changed, it means that it was modified during sync */
962 : /* this isn't a serious error, so we skip this block, and continue with others */
963 1 : error_on_this_block = 1;
964 1 : continue;
965 : } else { /* otherwise it's a BLK with silent error */
966 853 : unsigned diff = memdiff(hash, block->hash, BLOCK_HASH_SIZE);
967 853 : log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u/%u\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), file_pos, diff, BLOCK_HASH_SIZE * 8);
968 853 : log_error("Data error in file '%s' at position '%u', diff bits %u/%u\n", task->path, file_pos, diff, BLOCK_HASH_SIZE * 8);
969 :
970 : /* save the failed block for the fix */
971 853 : failed[failed_count].index = diskcur;
972 853 : failed[failed_count].size = read_size;
973 853 : failed[failed_count].block = block;
974 853 : ++failed_count;
975 :
976 : /* silent errors are very rare, and are not a signal that a disk */
977 : /* is going to fail. So, we just continue marking the block as bad */
978 : /* just like in scrub */
979 853 : ++silent_error;
980 853 : silent_error_on_this_block = 1;
981 853 : continue;
982 : }
983 : }
984 : } else {
985 : /* if until now the parity doesn't need to be updated */
986 94568 : if (!parity_needs_to_be_updated) {
987 : /* for sure it's a CHG block, because EMPTY are processed before with "continue" */
988 : /* and BLK and REP have "block_has_updated_hash()" as 1, and all the others */
989 : /* have "parity_needs_to_be_updated" already at 1 */
990 35228 : assert(block_state_get(block) == BLOCK_STATE_CHG);
991 :
992 : /* if the hash represents the data unequivocally */
993 35228 : if (hash_is_unique(block->hash)) {
994 : /* check if the hash is changed */
995 8554 : if (memcmp(hash, block->hash, BLOCK_HASH_SIZE) != 0) {
996 : /* the block is different, and we must update parity */
997 5961 : parity_needs_to_be_updated = 1;
998 : }
999 : } else {
1000 : /* if the hash is already invalid, we update parity */
1001 26674 : parity_needs_to_be_updated = 1;
1002 : }
1003 : }
1004 :
1005 : /* copy the hash in the block, but doesn't mark the block as hashed */
1006 : /* this allow in case of skipped block to do not save the failed computation */
1007 94568 : memcpy(block->hash, hash, BLOCK_HASH_SIZE);
1008 :
1009 : /* note that in case of rehash, this is the wrong hash, */
1010 : /* but it will be overwritten later */
1011 : }
1012 : }
1013 :
1014 : /* if we have only silent errors we can try to fix them on-the-fly */
1015 : /* note the the fix is not written to disk, but used only to */
1016 : /* compute the new parity */
1017 114756 : if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) {
1018 : unsigned failed_mac;
1019 853 : int something_to_recover = 0;
1020 :
1021 : /* sort the failed vector */
1022 : /* because with threads it may be in any order */
1023 : /* but RAID requires the indexes to be sorted */
1024 853 : qsort(failed, failed_count, sizeof(failed[0]), failed_compare_by_index);
1025 :
1026 : /* setup the blocks to recover */
1027 853 : failed_mac = 0;
1028 2559 : for (j = 0; j < failed_count; ++j) {
1029 1706 : unsigned char* block_buffer = buffer[failed[j].index];
1030 1706 : unsigned char* block_copy = copy[failed[j].index];
1031 1706 : unsigned block_state = block_state_get(failed[j].block);
1032 :
1033 : /* we try to recover only if at least one BLK is present */
1034 1706 : if (block_state == BLOCK_STATE_BLK)
1035 853 : something_to_recover = 1;
1036 :
1037 : /* save a copy of the content just read */
1038 : /* that it's going to be overwritten by the recovering function */
1039 1706 : memcpy(block_copy, block_buffer, state->block_size);
1040 :
1041 1706 : if (block_state == BLOCK_STATE_CHG
1042 853 : && hash_is_zero(failed[j].block->hash)
1043 : ) {
1044 : /* if the block was filled with 0, restore this state */
1045 : /* and avoid to recover it */
1046 0 : memset(block_buffer, 0, state->block_size);
1047 : } else {
1048 : /* if we have too many failures, we cannot recover */
1049 1706 : if (failed_mac >= state->level)
1050 0 : break;
1051 :
1052 : /* otherwise it has to be recovered */
1053 1706 : failed_map[failed_mac++] = failed[j].index;
1054 : }
1055 : }
1056 :
1057 : /* if we have something to recover and enough parity */
1058 853 : if (something_to_recover && j == failed_count) {
1059 : /* until now is misc */
1060 853 : state_usage_misc(state);
1061 :
1062 : /* read the parity */
1063 : /* we are sure that parity exists because */
1064 : /* we have at least one BLK block */
1065 5971 : for (l = 0; l < state->level; ++l) {
1066 5118 : ret = parity_read(&parity_handle[l], blockcur, buffer[diskmax + l], state->block_size, log_error);
1067 5118 : if (ret == -1) {
1068 : /* LCOV_EXCL_START */
1069 : if (errno == EIO) {
1070 : log_tag("parity_error:%u:%s: Read EIO error. %s\n", blockcur, lev_config_name(l), strerror(errno));
1071 : if (io_error >= state->opt.io_error_limit) {
1072 : log_fatal("DANGER! Unexpected input/output read error in the %s disk, it isn't possible to sync.\n", lev_name(l));
1073 : log_fatal("Ensure that disk '%s' is sane and can be read.\n", lev_config_name(l));
1074 : log_fatal("Stopping at block %u\n", blockcur);
1075 : ++io_error;
1076 : goto bail;
1077 : }
1078 :
1079 : log_error("Input/Output error in parity '%s' at position '%u'\n", lev_config_name(l), blockcur);
1080 : ++io_error;
1081 : io_error_on_this_block = 1;
1082 : continue;
1083 : }
1084 :
1085 : log_tag("parity_error:%u:%s: Read error. %s\n", blockcur, lev_config_name(l), strerror(errno));
1086 : log_fatal("WARNING! Unexpected read error in the %s disk, it isn't possible to sync.\n", lev_name(l));
1087 : log_fatal("Ensure that disk '%s' can be read.\n", lev_config_name(l));
1088 : log_fatal("Stopping at block %u\n", blockcur);
1089 : ++error;
1090 : goto bail;
1091 : /* LCOV_EXCL_STOP */
1092 : }
1093 :
1094 : /* until now is parity */
1095 5118 : state_usage_parity(state, &l, 1);
1096 : }
1097 :
1098 : /* if no error in parity read */
1099 853 : if (!io_error_on_this_block) {
1100 : /* try to fix the data */
1101 : /* note that this is a simple fix algorithm, that doesn't take into */
1102 : /* account the case of a wrong parity */
1103 : /* only 'fix' supports the most advanced fixing */
1104 853 : raid_rec(failed_mac, failed_map, diskmax, state->level, state->block_size, buffer);
1105 :
1106 : /* until now is raid */
1107 853 : state_usage_raid(state);
1108 :
1109 : /* check the result and prepare the data */
1110 5118 : for (j = 0; j < failed_count; ++j) {
1111 : unsigned char hash[HASH_MAX];
1112 1706 : unsigned char* block_buffer = buffer[failed[j].index];
1113 1706 : unsigned char* block_copy = copy[failed[j].index];
1114 1706 : unsigned block_state = block_state_get(failed[j].block);
1115 :
1116 1706 : if (block_state == BLOCK_STATE_BLK) {
1117 853 : unsigned size = failed[j].size;
1118 :
1119 : /* compute the hash of the recovered block */
1120 853 : if (rehash) {
1121 0 : memhash(state->prevhash, state->prevhashseed, hash, block_buffer, size);
1122 : } else {
1123 853 : memhash(state->hash, state->hashseed, hash, block_buffer, size);
1124 : }
1125 :
1126 : /* until now is hash */
1127 853 : state_usage_hash(state);
1128 :
1129 : /* if the hash doesn't match */
1130 853 : if (memcmp(hash, failed[j].block->hash, BLOCK_HASH_SIZE) != 0) {
1131 : /* we have not recovered */
1132 0 : break;
1133 : }
1134 :
1135 : /* pad with 0 if needed */
1136 853 : if (size < state->block_size)
1137 341 : memset(block_buffer + size, 0, state->block_size - size);
1138 : } else {
1139 : /* otherwise restore the content */
1140 : /* because we are not interested in the old state */
1141 : /* that it's recovered for CHG, REP and DELETED blocks */
1142 853 : memcpy(block_buffer, block_copy, state->block_size);
1143 : }
1144 : }
1145 :
1146 : /* if all is processed, we have fixed it */
1147 853 : if (j == failed_count)
1148 853 : fixed_error_on_this_block = 1;
1149 : }
1150 : }
1151 : }
1152 :
1153 : /* if we have read all the data required and it's correct, proceed with the parity */
1154 114756 : if (!error_on_this_block && !io_error_on_this_block
1155 114737 : && (!silent_error_on_this_block || fixed_error_on_this_block)
1156 : ) {
1157 : /* update the parity only if really needed */
1158 114737 : if (parity_needs_to_be_updated) {
1159 : /* compute the parity */
1160 112360 : raid_gen(diskmax, state->level, state->block_size, buffer);
1161 :
1162 : /* until now is raid */
1163 112360 : state_usage_raid(state);
1164 :
1165 : /* mark that the parity is going to be written */
1166 112360 : parity_going_to_be_updated = 1;
1167 : }
1168 :
1169 : /* for each disk, mark the blocks as processed */
1170 803159 : for (j = 0; j < diskmax; ++j) {
1171 : struct snapraid_block* block;
1172 :
1173 688422 : if (!handle[j].disk)
1174 0 : continue;
1175 :
1176 688422 : block = fs_par2block_find(handle[j].disk, blockcur);
1177 :
1178 688422 : if (block == BLOCK_NULL) {
1179 : /* nothing to do */
1180 81230 : continue;
1181 : }
1182 :
1183 : /* if it's a deleted block */
1184 607192 : if (block_state_get(block) == BLOCK_STATE_DELETED) {
1185 : /* the parity is now updated without this block, so it's now empty */
1186 10953 : fs_deallocate(handle[j].disk, blockcur);
1187 10953 : continue;
1188 : }
1189 :
1190 : /* now all the blocks have the hash and the parity computed */
1191 596239 : block_state_set(block, BLOCK_STATE_BLK);
1192 : }
1193 :
1194 : /* we update the info block only if we really have updated the parity */
1195 : /* because otherwise the time/justsynced info would be misleading as we didn't */
1196 : /* wrote the parity at this time */
1197 : /* we also update the info block only if no silent error was found */
1198 : /* because has no sense to refresh the time for data that we know bad */
1199 114737 : if (parity_needs_to_be_updated
1200 112360 : && !silent_error_on_this_block
1201 : ) {
1202 : /* if rehash is needed */
1203 112001 : if (rehash) {
1204 : /* store all the new hash already computed */
1205 32249 : for (j = 0; j < diskmax; ++j) {
1206 27642 : if (rehandle[j].block)
1207 27243 : memcpy(rehandle[j].block->hash, rehandle[j].hash, BLOCK_HASH_SIZE);
1208 : }
1209 : }
1210 :
1211 : /* update the time info of the block */
1212 : /* we are also clearing any previous bad and rehash flag */
1213 112001 : info_set(&state->infoarr, blockcur, info_make(now, 0, 0, 1));
1214 : }
1215 : }
1216 :
1217 : /* if a silent (even if corrected) or input/output error was found */
1218 : /* mark the block as bad to have check/fix to handle it */
1219 : /* because our correction is in memory only and not yet written */
1220 114756 : if (silent_error_on_this_block || io_error_on_this_block) {
1221 : /* set the error status keeping the other info */
1222 853 : info_set(&state->infoarr, blockcur, info_set_bad(info));
1223 : }
1224 :
1225 : /* finally schedule parity write */
1226 : /* Note that the calls to io_parity_write() are mandatory */
1227 : /* even if the parity doesn't need to be updated */
1228 : /* This because we want to keep track of the time usage */
1229 114756 : state_usage_misc(state);
1230 :
1231 : /* write start */
1232 114756 : io_write_preset(&io, blockcur, !parity_going_to_be_updated);
1233 :
1234 : /* write the parity */
1235 673187 : for (l = 0; l < state->level; ++l) {
1236 : unsigned levcur;
1237 :
1238 558431 : io_parity_write(&io, &levcur, waiting_map, &waiting_mac);
1239 :
1240 : /* until now is parity */
1241 558431 : state_usage_parity(state, waiting_map, waiting_mac);
1242 : }
1243 :
1244 : /* write finished */
1245 114756 : io_write_next(&io, blockcur, !parity_going_to_be_updated, writer_error);
1246 :
1247 : /* handle errors reported */
1248 573780 : for (j = 0; j < IO_WRITER_ERROR_MAX; ++j) {
1249 459024 : if (writer_error[j]) {
1250 0 : switch (j + IO_WRITER_ERROR_BASE) {
1251 : case TASK_STATE_IOERROR_CONTINUE :
1252 0 : ++io_error;
1253 0 : if (io_error >= state->opt.io_error_limit) {
1254 : /* LCOV_EXCL_START */
1255 : log_fatal("DANGER! Unexpected input/output write error in a parity disk, it isn't possible to sync.\n");
1256 : log_fatal("Stopping at block %u\n", blockcur);
1257 : goto bail;
1258 : /* LCOV_EXCL_STOP */
1259 : }
1260 0 : break;
1261 : case TASK_STATE_ERROR_CONTINUE :
1262 0 : ++error;
1263 0 : break;
1264 : case TASK_STATE_IOERROR :
1265 : /* LCOV_EXCL_START */
1266 : ++io_error;
1267 : goto bail;
1268 : /* LCOV_EXCL_STOP */
1269 : case TASK_STATE_ERROR :
1270 : /* LCOV_EXCL_START */
1271 : ++error;
1272 : goto bail;
1273 : /* LCOV_EXCL_STOP */
1274 : }
1275 : }
1276 : }
1277 :
1278 : /* mark the state as needing write */
1279 114756 : state->need_write = 1;
1280 :
1281 : /* count the number of processed block */
1282 114756 : ++countpos;
1283 :
1284 : /* progress */
1285 114756 : if (state_progress(state, &io, blockcur, countpos, countmax, countsize)) {
1286 : /* LCOV_EXCL_START */
1287 : break;
1288 : /* LCOV_EXCL_STOP */
1289 : }
1290 :
1291 : /* autosave */
1292 114756 : if ((state->autosave != 0
1293 26021 : && autosavedone >= autosavelimit /* if we have reached the limit */
1294 0 : && autosavemissing >= autosavelimit) /* if we have at least a full step to do */
1295 : /* or if we have a forced autosave at the specified block */
1296 114756 : || (state->opt.force_autosave_at != 0 && state->opt.force_autosave_at == blockcur)
1297 : ) {
1298 1 : autosavedone = 0; /* restart the counter */
1299 :
1300 : /* until now is misc */
1301 1 : state_usage_misc(state);
1302 :
1303 1 : state_progress_stop(state);
1304 :
1305 1 : msg_progress("Autosaving...\n");
1306 :
1307 : /* before writing the new content file we ensure that */
1308 : /* the parity is really written flushing the disk cache */
1309 7 : for (l = 0; l < state->level; ++l) {
1310 6 : ret = parity_sync(&parity_handle[l]);
1311 6 : if (ret == -1) {
1312 : /* LCOV_EXCL_START */
1313 : log_tag("parity_error:%u:%s: Sync error\n", blockcur, lev_config_name(l));
1314 : log_fatal("DANGER! Unexpected sync error in %s disk.\n", lev_name(l));
1315 : log_fatal("Ensure that disk '%s' is sane.\n", lev_config_name(l));
1316 : log_fatal("Stopping at block %u\n", blockcur);
1317 : ++error;
1318 : goto bail;
1319 : /* LCOV_EXCL_STOP */
1320 : }
1321 : }
1322 :
1323 : /* now we can safely write the content file */
1324 1 : state_write(state);
1325 :
1326 1 : state_progress_restart(state);
1327 :
1328 : /* drop until now */
1329 1 : state_usage_waste(state);
1330 : }
1331 114756 : }
1332 :
1333 : end:
1334 83 : state_progress_end(state, countpos, countmax, countsize);
1335 :
1336 83 : state_usage_print(state);
1337 :
1338 : /* before returning we ensure that */
1339 : /* the parity is really written flushing the disk cache */
1340 546 : for (l = 0; l < state->level; ++l) {
1341 463 : ret = parity_sync(&parity_handle[l]);
1342 463 : if (ret == -1) {
1343 : /* LCOV_EXCL_START */
1344 : log_tag("parity_error:%u:%s: Sync error\n", blockcur, lev_config_name(l));
1345 : log_fatal("DANGER! Unexpected sync error in %s disk.\n", lev_name(l));
1346 : log_fatal("Ensure that disk '%s' is sane.\n", lev_config_name(l));
1347 : log_fatal("Stopping at block %u\n", blockcur);
1348 : ++error;
1349 : goto bail;
1350 : /* LCOV_EXCL_STOP */
1351 : }
1352 : }
1353 :
1354 83 : if (error || silent_error || io_error) {
1355 12 : msg_status("\n");
1356 12 : msg_status("%8u file errors\n", error);
1357 12 : msg_status("%8u io errors\n", io_error);
1358 12 : msg_status("%8u data errors\n", silent_error);
1359 : } else {
1360 : /* print the result only if processed something */
1361 71 : if (countpos != 0)
1362 53 : msg_status("Everything OK\n");
1363 : }
1364 :
1365 83 : if (error)
1366 11 : log_fatal("WARNING! Unexpected file errors!\n");
1367 83 : if (io_error)
1368 0 : log_fatal("DANGER! Unexpected input/output errors! The failing blocks are now marked as bad!\n");
1369 83 : if (silent_error)
1370 1 : log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
1371 83 : if (io_error || silent_error) {
1372 1 : log_fatal("Use 'snapraid status' to list the bad blocks.\n");
1373 1 : log_fatal("Use 'snapraid -e fix' to recover.\n");
1374 : }
1375 :
1376 83 : log_tag("summary:error_file:%u\n", error);
1377 83 : log_tag("summary:error_io:%u\n", io_error);
1378 83 : log_tag("summary:error_data:%u\n", silent_error);
1379 83 : if (error + silent_error + io_error == 0)
1380 71 : log_tag("summary:exit:ok\n");
1381 : else
1382 12 : log_tag("summary:exit:error\n");
1383 83 : log_flush();
1384 :
1385 : bail:
1386 : /* stop all the worker threads */
1387 83 : io_stop(&io);
1388 :
1389 581 : for (j = 0; j < diskmax; ++j) {
1390 498 : struct snapraid_file* file = handle[j].file;
1391 498 : struct snapraid_disk* disk = handle[j].disk;
1392 498 : ret = handle_close(&handle[j]);
1393 498 : if (ret == -1) {
1394 : /* LCOV_EXCL_START */
1395 : log_tag("error:%u:%s:%s: Close error. %s\n", blockcur, disk->name, esc_tag(file->sub, esc_buffer), strerror(errno));
1396 : log_fatal("DANGER! Unexpected close error in a data disk.\n");
1397 : ++error;
1398 : /* continue, as we are already exiting */
1399 : /* LCOV_EXCL_STOP */
1400 : }
1401 : }
1402 :
1403 83 : free(handle);
1404 83 : free(zero_alloc);
1405 83 : free(copy_alloc);
1406 83 : free(copy);
1407 83 : free(rehandle_alloc);
1408 83 : free(failed);
1409 83 : free(failed_map);
1410 83 : free(waiting_map);
1411 83 : io_done(&io);
1412 :
1413 83 : if (state->opt.expect_recoverable) {
1414 1 : if (error + silent_error + io_error == 0)
1415 0 : return -1;
1416 : } else {
1417 82 : if (error + silent_error + io_error != 0)
1418 11 : return -1;
1419 : }
1420 72 : return 0;
1421 : }
1422 :
1423 87 : int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount)
1424 : {
1425 : block_off_t blockmax;
1426 : block_off_t used_paritymax;
1427 : block_off_t file_paritymax;
1428 : data_off_t size;
1429 : int ret;
1430 : struct snapraid_parity_handle parity_handle[LEV_MAX];
1431 : unsigned unrecoverable_error;
1432 : unsigned l;
1433 87 : int skip_sync = 0;
1434 :
1435 87 : msg_progress("Initializing...\n");
1436 :
1437 87 : blockmax = parity_allocated_size(state);
1438 87 : size = blockmax * (data_off_t)state->block_size;
1439 :
1440 : /* minimum size of the parity files we expect */
1441 87 : used_paritymax = parity_used_size(state);
1442 :
1443 : /* effective size of the parity files */
1444 87 : file_paritymax = 0;
1445 :
1446 87 : if (blockstart > blockmax) {
1447 : /* LCOV_EXCL_START */
1448 : log_fatal("Error in the starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
1449 : exit(EXIT_FAILURE);
1450 : /* LCOV_EXCL_STOP */
1451 : }
1452 :
1453 : /* adjust the number of block to process */
1454 87 : if (blockcount != 0 && blockstart + blockcount < blockmax) {
1455 6 : blockmax = blockstart + blockcount;
1456 : }
1457 :
1458 574 : for (l = 0; l < state->level; ++l) {
1459 : data_off_t out_size;
1460 : block_off_t parityblocks;
1461 :
1462 : /* create the file and open for writing */
1463 487 : ret = parity_create(&parity_handle[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
1464 487 : if (ret == -1) {
1465 : /* LCOV_EXCL_START */
1466 : log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
1467 : exit(EXIT_FAILURE);
1468 : /* LCOV_EXCL_STOP */
1469 : }
1470 :
1471 : /* number of block in the parity file */
1472 487 : parity_size(&parity_handle[l], &out_size);
1473 487 : parityblocks = out_size / state->block_size;
1474 :
1475 : /* if the file is too small */
1476 487 : if (parityblocks < used_paritymax) {
1477 0 : log_fatal("WARNING! The %s parity has data only %u blocks instead of %u.\n", lev_name(l), parityblocks, used_paritymax);
1478 : }
1479 :
1480 : /* keep the smallest parity number of blocks */
1481 487 : if (l == 0 || file_paritymax > parityblocks)
1482 87 : file_paritymax = parityblocks;
1483 : }
1484 :
1485 : /* if we do a full parity realloc or computation, having a wrong parity size is expected */
1486 87 : if (!state->opt.force_realloc && !state->opt.force_full) {
1487 : /* if the parities are too small */
1488 80 : if (file_paritymax < used_paritymax) {
1489 : /* LCOV_EXCL_START */
1490 : log_fatal("DANGER! One or more the parity files are smaller than expected!\n");
1491 : if (file_paritymax != 0) {
1492 : log_fatal("If this happens because you are using an old content file,\n");
1493 : log_fatal("you can 'sync' anyway using 'snapraid --force-full sync'\n");
1494 : log_fatal("to force a full rebuild of the parity.\n");
1495 : } else {
1496 : log_fatal("It's possible that the parity disks are not mounted.\n");
1497 : log_fatal("If instead you are adding a new parity level, you can 'sync' using\n");
1498 : log_fatal("'snapraid --force-full sync' to force a full rebuild of the parity.\n");
1499 : }
1500 : exit(EXIT_FAILURE);
1501 : /* LCOV_EXCL_STOP */
1502 : }
1503 : }
1504 :
1505 87 : unrecoverable_error = 0;
1506 :
1507 87 : if (state->opt.prehash) {
1508 7 : msg_progress("Hashing...\n");
1509 :
1510 7 : ret = state_hash_process(state, blockstart, blockmax, &skip_sync);
1511 7 : if (ret == -1) {
1512 : /* LCOV_EXCL_START */
1513 : ++unrecoverable_error;
1514 : /* continue, in case also doing the sync if ::skip_sync is not set */
1515 : /* LCOV_EXCL_STOP */
1516 : }
1517 : }
1518 :
1519 87 : if (!skip_sync) {
1520 86 : msg_progress("Resizing...\n");
1521 :
1522 : /* now change the size of all parities */
1523 567 : for (l = 0; l < state->level; ++l) {
1524 : int is_modified;
1525 :
1526 : /* change the size of the parity file, truncating or extending it */
1527 : /* from this point all the DELETED blocks after the end of the parity are invalid */
1528 : /* and they are automatically removed when we save the new content file */
1529 481 : ret = parity_chsize(&parity_handle[l], &state->parity[l], &is_modified, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder);
1530 481 : if (ret == -1) {
1531 : /* LCOV_EXCL_START */
1532 : data_off_t out_size;
1533 : parity_size(&parity_handle[l], &out_size);
1534 : parity_overflow(state, out_size);
1535 : log_fatal("WARNING! Without an unsable %s file, it isn't possible to sync.\n", lev_name(l));
1536 : exit(EXIT_FAILURE);
1537 : /* LCOV_EXCL_STOP */
1538 : }
1539 :
1540 481 : if (is_modified)
1541 175 : state->need_write = 1;
1542 : }
1543 :
1544 : /* after resizing parity files, refresh again the free info */
1545 86 : state_refresh(state);
1546 :
1547 : /**
1548 : * Save the new state before the sync but after the hashing phase
1549 : *
1550 : * This allows to recover after an aborted sync, and at the same time
1551 : * it allows to recover broken copied/moved files identified in the
1552 : * hashing phase.
1553 : *
1554 : * For example, think at this case:
1555 : * - Add some files at the array
1556 : * - Run a sync command, it will recompute the parity adding the new files
1557 : * - Abort the sync command before it stores the new content file
1558 : * - Delete the not yet synced files from the array
1559 : * - Run a new sync command
1560 : *
1561 : * The sync command has no way to know that the parity file was modified
1562 : * because the files triggering these changes are now deleted and they aren't
1563 : * listed in the content file.
1564 : * Instead, saving the new content file in advance, keeps track of all the parity
1565 : * that may be modified.
1566 : */
1567 86 : if (!state->opt.skip_content_write) {
1568 86 : if (state->need_write)
1569 79 : state_write(state);
1570 : } else {
1571 0 : log_fatal("WARNING! Skipped state write for --test-skip-content-write option.\n");
1572 : }
1573 :
1574 86 : msg_progress("Syncing...\n");
1575 :
1576 : /* skip degenerated cases of empty parity, or skipping all */
1577 86 : if (blockstart < blockmax) {
1578 83 : ret = state_sync_process(state, parity_handle, blockstart, blockmax);
1579 83 : if (ret == -1) {
1580 : /* LCOV_EXCL_START */
1581 : ++unrecoverable_error;
1582 : /* continue, as we are already exiting */
1583 : /* LCOV_EXCL_STOP */
1584 : }
1585 : } else {
1586 3 : msg_status("Nothing to do\n");
1587 : }
1588 : }
1589 :
1590 574 : for (l = 0; l < state->level; ++l) {
1591 487 : ret = parity_close(&parity_handle[l]);
1592 487 : if (ret == -1) {
1593 : /* LCOV_EXCL_START */
1594 : log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
1595 : ++unrecoverable_error;
1596 : /* continue, as we are already exiting */
1597 : /* LCOV_EXCL_STOP */
1598 : }
1599 : }
1600 :
1601 : /* abort if required */
1602 87 : if (unrecoverable_error != 0)
1603 12 : return -1;
1604 75 : return 0;
1605 : }
1606 :
|
