Line data Source code
1 : /* +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 : University of Illinois Open Source License
3 : Copyright 2003 Theoretical and Computational Biophysics Group,
4 : All rights reserved.
5 :
6 : Developed by: Theoretical and Computational Biophysics Group
7 : University of Illinois at Urbana-Champaign
8 : http://www.ks.uiuc.edu/
9 :
10 : Permission is hereby granted, free of charge, to any person obtaining a copy of
11 : this software and associated documentation files (the Software), to deal with
12 : the Software without restriction, including without limitation the rights to
13 : use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
14 : of the Software, and to permit persons to whom the Software is furnished to
15 : do so, subject to the following conditions:
16 :
17 : Redistributions of source code must retain the above copyright notice,
18 : this list of conditions and the following disclaimers.
19 :
20 : Redistributions in binary form must reproduce the above copyright notice,
21 : this list of conditions and the following disclaimers in the documentation
22 : and/or other materials provided with the distribution.
23 :
24 : Neither the names of Theoretical and Computational Biophysics Group,
25 : University of Illinois at Urbana-Champaign, nor the names of its contributors
26 : may be used to endorse or promote products derived from this Software without
27 : specific prior written permission.
28 :
29 : THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30 : IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31 : FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
32 : THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
33 : OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
34 : ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
35 : OTHER DEALINGS WITH THE SOFTWARE.
36 : +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
37 : #ifndef __PLUMED_molfile_fastio_h
38 : #define __PLUMED_molfile_fastio_h
39 : /***************************************************************************
40 : *cr
41 : *cr (C) Copyright 1995-2016 The Board of Trustees of the
42 : *cr University of Illinois
43 : *cr All Rights Reserved
44 : *cr
45 : ***************************************************************************/
46 : /***************************************************************************
47 : * RCS INFORMATION:
48 : *
49 : * $RCSfile: fastio.h,v $
50 : * $Author: johns $ $Locker: $ $State: Exp $
51 : * $Revision: 1.34 $ $Date: 2016/11/28 05:01:53 $
52 : *
53 : ***************************************************************************
54 : * DESCRIPTION:
55 : * This is a simple abstraction layer for system-dependent I/O calls
56 : * that allow plugins to do binary I/O using the fastest possible method.
57 : *
58 : * This code is intended for use by binary trajectory reader plugins that
59 : * work with multi-gigabyte data sets, reading only binary data.
60 : *
61 : ***************************************************************************/
62 :
63 : #define FIO_READ 0x01
64 : #define FIO_WRITE 0x02
65 : #define FIO_DIRECT 0x04 /* emulate Unix O_DIRECT flag */
66 :
67 : /* Compiling on windows */
68 : #if defined(_MSC_VER) || defined(__MINGW32__)
69 :
70 : #if 1
71 : /* use native Windows I/O calls */
72 : #define FASTIO_NATIVEWIN32 1
73 :
74 : #include <stdio.h>
75 : #include <string.h>
76 : #include <windows.h>
77 :
78 : namespace PLMD{
79 : namespace molfile{
80 :
81 : typedef HANDLE fio_fd;
82 : typedef LONGLONG fio_size_t;
83 : typedef void * fio_caddr_t;
84 :
85 : typedef struct {
86 : fio_caddr_t iov_base;
87 : int iov_len;
88 : } fio_iovec;
89 :
90 :
91 : #define FIO_SEEK_CUR FILE_CURRENT
92 : #define FIO_SEEK_SET FILE_BEGIN
93 : #define FIO_SEEK_END FILE_END
94 :
95 : static int fio_win32convertfilename(const char *filename, char *newfilename, int maxlen) {
96 : int i;
97 : int len=strlen(filename);
98 :
99 : if ((len + 1) >= maxlen)
100 : return -1;
101 :
102 : for (i=0; i<len; i++) {
103 : if (filename[i] == '/')
104 : newfilename[i] = '\\';
105 : else
106 : newfilename[i] = filename[i];
107 : }
108 : newfilename[len] = '\0'; /* NUL terminate the string */
109 :
110 : return 0;
111 : }
112 :
113 : static int fio_open(const char *filename, int mode, fio_fd *fd) {
114 : HANDLE fp;
115 : char winfilename[8192];
116 : DWORD access;
117 : DWORD sharing;
118 : LPSECURITY_ATTRIBUTES security;
119 : DWORD createmode;
120 : DWORD flags;
121 :
122 : if (fio_win32convertfilename(filename, winfilename, sizeof(winfilename)))
123 : return -1;
124 :
125 : access = 0;
126 : if (mode & FIO_READ)
127 : access |= GENERIC_READ;
128 : if (mode & FIO_WRITE)
129 : access |= GENERIC_WRITE;
130 : #if 0
131 : access = FILE_ALL_ACCESS; /* XXX hack if above modes fail */
132 : #endif
133 : #if 1
134 : if (mode & FIO_DIRECT)
135 : flags = FILE_FLAG_NO_BUFFERING;
136 : else
137 : flags = FILE_ATTRIBUTE_NORMAL;
138 : #else
139 : if (mode & FIO_DIRECT)
140 : return -1; /* not supported yet */
141 : #endif
142 :
143 : sharing = 0; /* disallow sharing with other processes */
144 : security = NULL; /* child processes don't inherit anything */
145 :
146 : /* since we never append, blow away anything that's already there */
147 : if (mode & FIO_WRITE)
148 : createmode = CREATE_ALWAYS;
149 : else
150 : createmode = OPEN_EXISTING;
151 :
152 : fp = CreateFile(winfilename, access, sharing, security,
153 : createmode, flags, NULL);
154 :
155 : if (fp == NULL) {
156 : return -1;
157 : } else {
158 : *fd = fp;
159 : return 0;
160 : }
161 : }
162 :
163 :
164 : static int fio_fclose(fio_fd fd) {
165 : BOOL rc;
166 : rc = CloseHandle(fd);
167 : if (rc)
168 : return 0;
169 : else
170 : return -1;
171 : }
172 :
173 : static fio_size_t fio_fread(void *ptr, fio_size_t size,
174 : fio_size_t nitems, fio_fd fd) {
175 : BOOL rc;
176 : DWORD len;
177 : DWORD readlen;
178 :
179 : len = size * nitems;
180 :
181 : rc = ReadFile(fd, ptr, len, &readlen, NULL);
182 : if (rc) {
183 : if (readlen == len)
184 : return nitems;
185 : else
186 : return 0;
187 : } else {
188 : return 0;
189 : }
190 : }
191 :
192 : static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
193 : int i;
194 : fio_size_t len = 0;
195 :
196 : for (i=0; i<iovcnt; i++) {
197 : fio_size_t rc = fio_fread(iov[i].iov_base, iov[i].iov_len, 1, fd);
198 : if (rc != 1)
199 : break;
200 : len += iov[i].iov_len;
201 : }
202 :
203 : return len;
204 : }
205 :
206 : static fio_size_t fio_fwrite(void *ptr, fio_size_t size,
207 : fio_size_t nitems, fio_fd fd) {
208 : BOOL rc;
209 : DWORD len;
210 : DWORD writelen;
211 :
212 : len = size * nitems;
213 :
214 : rc = WriteFile(fd, ptr, len, &writelen, NULL);
215 : if (rc) {
216 : if (writelen == len)
217 : return nitems;
218 : else
219 : return 0;
220 : } else {
221 : return 0;
222 : }
223 : }
224 :
225 : static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
226 : #if 1
227 : /* code that works with older MSVC6 compilers */
228 : LONGLONG finaloffset;
229 : LARGE_INTEGER bigint;
230 : LARGE_INTEGER finalint;
231 :
232 : bigint.QuadPart = offset;
233 : finalint = bigint; /* set the high part, which will be overwritten */
234 : finalint.LowPart = SetFilePointer(fd, bigint.LowPart, &finalint.HighPart, whence);
235 : if (finalint.LowPart == -1) {
236 : /* if (finalint.LowPart == INVALID_SET_FILE_POINTER) { */
237 : /* INVALID_SET_FILE_POINTER is a possible "ok" low order result when */
238 : /* working with 64-bit offsets, so we have to also check the system */
239 : /* error value for this thread to be sure */
240 : if (GetLastError() != ERROR_SUCCESS) {
241 : return -1;
242 : }
243 : }
244 :
245 : finaloffset = finalint.QuadPart;
246 : return 0;
247 : #else
248 : BOOL rc;
249 : LONGLONG finaloffset;
250 :
251 : /* SetFilePointerEx() only exists with new .NET compilers */
252 : rc = SetFilePointerEx(fd, offset, &finaloffset, whence);
253 :
254 : if (rc)
255 : return 0;
256 : else
257 : return -1;
258 : #endif
259 : }
260 :
261 : static fio_size_t fio_ftell(fio_fd fd) {
262 : /* code that works with older MSVC6 compilers */
263 : LONGLONG finaloffset;
264 : LARGE_INTEGER bigint;
265 : LARGE_INTEGER finalint;
266 :
267 : bigint.QuadPart = 0;
268 : finalint = bigint; /* set the high part, which will be overwritten */
269 :
270 : finalint.LowPart = SetFilePointer(fd, bigint.LowPart, &finalint.HighPart, FILE_CURRENT);
271 : if (finalint.LowPart == -1) {
272 : /* if (finalint.LowPart == INVALID_SET_FILE_POINTER) { */
273 : /* INVALID_SET_FILE_POINTER is a possible "ok" low order result when */
274 : /* working with 64-bit offsets, so we have to also check the system */
275 : /* error value for this thread to be sure */
276 : if (GetLastError() != ERROR_SUCCESS) {
277 : return -1;
278 : }
279 : }
280 :
281 : finaloffset = finalint.QuadPart;
282 :
283 : return finaloffset;
284 : }
285 :
286 :
287 : #else
288 :
289 : /* Version for machines with plain old ANSI C */
290 :
291 : #include <stdio.h>
292 : #include <string.h>
293 :
294 : typedef FILE * fio_fd;
295 : typedef size_t fio_size_t; /* MSVC doesn't uinversally support ssize_t */
296 : typedef void * fio_caddr_t; /* MSVC doesn't universally support caddr_t */
297 :
298 : typedef struct {
299 : fio_caddr_t iov_base;
300 : int iov_len;
301 : } fio_iovec;
302 :
303 : #define FIO_SEEK_CUR SEEK_CUR
304 : #define FIO_SEEK_SET SEEK_SET
305 : #define FIO_SEEK_END SEEK_END
306 :
307 : static int fio_open(const char *filename, int mode, fio_fd *fd) {
308 : char * modestr;
309 : FILE *fp;
310 :
311 : if (mode & FIO_READ)
312 : modestr = "rb";
313 :
314 : if (mode & FIO_WRITE)
315 : modestr = "wb";
316 :
317 : if (mode & FIO_DIRECT)
318 : return -1; /* not supported yet */
319 :
320 : fp = fopen(filename, modestr);
321 : if (fp == NULL) {
322 : return -1;
323 : } else {
324 : *fd = fp;
325 : return 0;
326 : }
327 : }
328 :
329 : static int fio_fclose(fio_fd fd) {
330 : return fclose(fd);
331 : }
332 :
333 : static fio_size_t fio_fread(void *ptr, fio_size_t size,
334 : fio_size_t nitems, fio_fd fd) {
335 : return fread(ptr, size, nitems, fd);
336 : }
337 :
338 : static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
339 : int i;
340 : fio_size_t len = 0;
341 :
342 : for (i=0; i<iovcnt; i++) {
343 : fio_size_t rc = fread(iov[i].iov_base, iov[i].iov_len, 1, fd);
344 : if (rc != 1)
345 : break;
346 : len += iov[i].iov_len;
347 : }
348 :
349 : return len;
350 : }
351 :
352 : static fio_size_t fio_fwrite(void *ptr, fio_size_t size,
353 : fio_size_t nitems, fio_fd fd) {
354 : return fwrite(ptr, size, nitems, fd);
355 : }
356 :
357 : static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
358 : return fseek(fd, offset, whence);
359 : }
360 :
361 : static fio_size_t fio_ftell(fio_fd fd) {
362 : return ftell(fd);
363 : }
364 : #endif /* plain ANSI C */
365 :
366 : }
367 : }
368 :
369 : #else
370 :
371 : /* Version for UNIX machines */
372 : #if defined(__linux)
373 : #ifndef _GNU_SOURCE
374 : #define _GNU_SOURCE /* required for O_DIRECT */
375 : #endif
376 : #endif
377 : #include <unistd.h>
378 : #include <stdio.h>
379 : #include <sys/types.h>
380 : #include <sys/stat.h>
381 : #include <fcntl.h>
382 : #include <string.h>
383 :
384 : namespace PLMD{
385 : namespace molfile{
386 :
387 : typedef int fio_fd;
388 : typedef off_t fio_size_t; /* off_t is 64-bits with LFS builds */
389 :
390 : /*
391 : * Enable use of kernel readv() if available and reliable
392 : *
393 : * Note: Some Linux implementations incorporate readv() code in libc
394 : * that does userspace copying of I/O vectors to internal temporary
395 : * buffers in order to meet the atomicity requirements of the POSIX standard.
396 : * Such copies make the use of vectorized I/O APIs much less useful for
397 : * large trajectory files because the internal buffer allocations can fail
398 : * badly when performing large aggregate I/O operations. It may be that
399 : * other implementations of vector I/O have similar problems, and in these
400 : * cases it is probably best not to use it at all, and to fall back to
401 : * non-vectorized I/O APIs to avoid such extra copies.
402 : */
403 : #if defined(__sun) || defined(__APPLE_CC__) || defined(__linux)
404 : #define USE_KERNEL_READV 1
405 : #endif
406 :
407 : typedef void * fio_caddr_t;
408 :
409 : }
410 : }
411 :
412 : #if defined(USE_KERNEL_READV)
413 : #include <errno.h>
414 : #include <sys/uio.h>
415 : namespace PLMD{
416 : namespace molfile{
417 : typedef struct iovec fio_iovec;
418 : }
419 : }
420 : #else
421 :
422 : namespace PLMD{
423 : namespace molfile{
424 : typedef struct {
425 : fio_caddr_t iov_base;
426 : int iov_len;
427 : } fio_iovec;
428 : }
429 : }
430 : #endif
431 :
432 : #define FIO_SEEK_CUR SEEK_CUR
433 : #define FIO_SEEK_SET SEEK_SET
434 : #define FIO_SEEK_END SEEK_END
435 :
436 : namespace PLMD{
437 : namespace molfile{
438 :
439 1 : static int fio_open(const char *filename, int mode, fio_fd *fd) {
440 : int nfd;
441 : int oflag = 0;
442 :
443 : if (mode & FIO_READ)
444 : oflag = O_RDONLY;
445 :
446 1 : if (mode & FIO_WRITE)
447 : oflag = O_WRONLY | O_CREAT | O_TRUNC;
448 :
449 : #if defined(__linux)
450 : /* enable direct I/O, requires block-aligned buffers and I/O sizes */
451 1 : if (mode & FIO_DIRECT)
452 0 : oflag |= O_DIRECT;
453 : #else
454 : if (mode & FIO_DIRECT)
455 : return -1; /* not supported yet */
456 : #endif
457 :
458 : nfd = open(filename, oflag, 0666);
459 1 : if (nfd < 0) {
460 : return -1;
461 : } else {
462 1 : *fd = nfd;
463 1 : return 0;
464 : }
465 : }
466 :
467 1 : static int fio_fclose(fio_fd fd) {
468 1 : return close(fd);
469 : }
470 :
471 219 : static fio_size_t fio_fread(void *ptr, fio_size_t size,
472 : fio_size_t nitems, fio_fd fd) {
473 : fio_size_t i;
474 : fio_size_t len = 0;
475 : fio_size_t cnt = 0;
476 :
477 : #if 1
478 : /*
479 : * On Linux individual calls to read() can end up doing short reads when
480 : * reading more than 2GB in a single read call, even on 64-bit machines.
481 : * For large structures, e.g. 240M-atoms or larger, we have to use a loop
482 : * to continue reading into the memory buffer until completion.
483 : */
484 438 : for (i=0; i<nitems; i++) {
485 : fio_size_t szleft = size;
486 : fio_size_t rc = 0;
487 438 : for (szleft=size; szleft > 0; szleft -= rc) {
488 219 : rc = read(fd, ((char*) ptr) + (cnt*size) + (size-szleft), szleft);
489 219 : if (rc == 0) {
490 : return cnt; /* end of file scenario */
491 : }
492 : // if (rc != szleft) {
493 : // printf("fio_fread(): rc %ld sz: %ld\n", rc, szleft);
494 : // }
495 219 : if (rc < 0) {
496 : printf("fio_fread(): rc %ld sz: %ld\n", rc, size);
497 0 : perror(" perror fio_fread(): ");
498 0 : break;
499 : }
500 : }
501 : len += rc;
502 219 : cnt++;
503 : }
504 : #else
505 : for (i=0; i<nitems; i++) {
506 : fio_size_t rc = read(fd, (void*) (((char *) ptr) + (cnt * size)), size);
507 : if (rc != size) {
508 : // printf("fio_fread(): rc %ld sz: %ld\n", rc, size);
509 : // perror(" perror fio_fread(): ");
510 : break;
511 : }
512 : len += rc;
513 : cnt++;
514 : }
515 : #endif
516 :
517 : return cnt;
518 : }
519 :
520 : static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
521 : fio_size_t len;
522 : int i;
523 :
524 : #if 0
525 : fio_size_t tlen;
526 : for (tlen=0,i=0; i<iovcnt; i++) {
527 : tlen += iov[i].iov_len;
528 : }
529 :
530 : #if defined(USE_KERNEL_READV)
531 : len = readv(fd, iov, iovcnt);
532 : if (len != tlen) {
533 : printf("fio_readv(): readv() rc: %ld sz: %ld\n", len, tlen);
534 : printf("fio_readv(): readv() errno %d\n", errno);
535 : }
536 :
537 : if ((len < 0 && errno == ENOSYS) ||
538 : (len != tlen && errno == EINVAL))
539 : #endif
540 : {
541 : /* XXX this loop doesn't meet the atomicity requirements of
542 : * real POSIX readv(), since we don't need that feature
543 : */
544 : len = 0;
545 : for (i=0; i<iovcnt; i++) {
546 : void *ptr = iov[i].iov_base;
547 : fio_size_t sz = iov[i].iov_len;
548 : fio_size_t szleft = sz;
549 : fio_size_t rc=0;
550 :
551 : for (szleft=sz; szleft > 0; szleft -= rc) {
552 : rc = read(fd, ((char*) ptr)+(sz-szleft), szleft);
553 : if (rc == 0) {
554 : return len; /* end of file scenario */
555 : }
556 : if (rc != szleft) {
557 : printf("fio_readv(): read() rc %ld sz: %ld\n", rc, szleft);
558 : }
559 : if (rc < 0) {
560 : printf("fio_readv(): read() rc %ld sz: %ld\n", rc, szleft);
561 : perror(" perror fio_readv(): ");
562 : break;
563 : }
564 : }
565 : len += iov[i].iov_len;
566 : }
567 : }
568 : #else
569 : #if defined(USE_KERNEL_READV)
570 : len = readv(fd, iov, iovcnt);
571 : if (len < 0 && errno == ENOSYS)
572 : #endif
573 : {
574 : /* XXX this loop doesn't meet the atomicity requirements of
575 : * real POSIX readv(), since we don't need that feature
576 : */
577 : len = 0;
578 : for (i=0; i<iovcnt; i++) {
579 : fio_size_t rc = read(fd, iov[i].iov_base, iov[i].iov_len);
580 : if (rc != iov[i].iov_len)
581 : break;
582 : len += iov[i].iov_len;
583 : }
584 : }
585 : #endif
586 :
587 : return len;
588 : }
589 :
590 0 : static fio_size_t fio_fwrite(void *ptr, fio_size_t size,
591 : fio_size_t nitems, fio_fd fd) {
592 : fio_size_t i;
593 : fio_size_t len = 0;
594 : fio_size_t cnt = 0;
595 :
596 : #if 1
597 : /*
598 : * On Linux individual calls to write() can end up doing short writes when
599 : * writing more than 2GB in a single write call, even on 64-bit machines.
600 : * For large structures, e.g. 240M-atoms or larger, we have to use a loop
601 : * to continue writing the memory buffer until completion.
602 : */
603 : int writecalls=0;
604 0 : for (i=0; i<nitems; i++) {
605 : fio_size_t szleft = size;
606 : fio_size_t rc = 0;
607 0 : for (szleft=size; szleft > 0; szleft -= rc) {
608 : fio_size_t writesz = szleft;
609 :
610 : #if 0
611 : /* On some kernel versions write calls beyond 2GB may not do */
612 : /* a partial write and may just return an error immediately. */
613 : /* Clamp maximum write size to 1GB per write call. */
614 : if (writesz > (1024L * 1024L * 1024L))
615 : writesz = (1024L * 1024L * 1024L);
616 : #endif
617 :
618 0 : writecalls++;
619 0 : rc = write(fd, ((char*) ptr)+(size-szleft), writesz);
620 0 : if (rc < 0) {
621 : printf("fio_fwrite(): rc %ld sz: %ld szleft: %ld calls: %d\n",
622 : rc, size, szleft, writecalls);
623 0 : perror(" perror fio_fwrite(): ");
624 0 : return cnt;
625 : }
626 : }
627 : len += rc;
628 0 : cnt++;
629 : }
630 : #else
631 : for (i=0; i<nitems; i++) {
632 : fio_size_t rc = write(fd, ptr, size);
633 : if (rc != size) {
634 : printf("fio_fwrite(): rc %ld sz: %ld\n", rc, size);
635 : perror(" perror fio_fwrite(): ");
636 : break;
637 : }
638 : len += rc;
639 : cnt++;
640 : }
641 : #endif
642 :
643 : return cnt;
644 : }
645 :
646 2 : static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
647 2 : if (lseek(fd, offset, whence) >= 0)
648 : return 0; /* success (emulate behavior of fseek) */
649 : else
650 0 : return -1; /* failure (emulate behavior of fseek) */
651 : }
652 :
653 1 : static fio_size_t fio_ftell(fio_fd fd) {
654 1 : return lseek(fd, 0, SEEK_CUR);
655 : }
656 :
657 : }
658 : }
659 :
660 : #endif
661 :
662 : namespace PLMD{
663 : namespace molfile{
664 :
665 :
666 : /* higher level routines that are OS independent */
667 :
668 0 : static int fio_write_int32(fio_fd fd, int i) {
669 0 : return (fio_fwrite(&i, 4, 1, fd) != 1);
670 : }
671 :
672 : static int fio_read_int32(fio_fd fd, int *i) {
673 : return (fio_fread(i, 4, 1, fd) != 1);
674 : }
675 :
676 : static int fio_write_str(fio_fd fd, const char *str) {
677 : int len = strlen(str);
678 : return (fio_fwrite((void *) str, len, 1, fd) != 1);
679 : }
680 :
681 : }
682 : }
683 : #endif
|
