brukeropus.file.parse

  1import os, struct, errno
  2import numpy as np
  3from brukeropus.file.constants import STRUCT_3D_INFO_BLOCK, SUBREPORT_TYPE_FMT
  4
  5
  6__docformat__ = "google"
  7
  8
  9def read_opus_file_bytes(filepath) -> bytes:
 10    '''Returns `bytes` of an OPUS file specified by `filepath` (or `None`).
 11
 12    Function determines if `filepath` points to an OPUS file by reading the first four bytes which are always the same
 13    for OPUS files.  If `filepath` is not a file, or points to a non-OPUS file, the function returns `None`.  Otherwise
 14    the function returns the entire file as raw `bytes`.
 15
 16    Args:
 17        filepath (str or Path): full filepath to OPUS file
 18
 19    Returns:
 20        **filebytes (bytes):** raw bytes of OPUS file or `None` (if filepath does not point to an OPUS file)
 21    '''
 22    filebytes = None
 23    if os.path.isfile(filepath):
 24        with open(filepath, 'rb') as f:
 25            try:
 26                first_four = f.read(4)
 27                if first_four == b'\n\n\xfe\xfe':
 28                    filebytes = first_four + f.read()
 29            except:
 30                pass # Empty file (or file with fewer than 4 bytes)
 31    else:
 32        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filepath)
 33    return filebytes
 34
 35
 36def get_block_type(type_int: int) -> tuple:
 37    '''Converts an int32 block type code to a six-integer tuple `block_type`.
 38
 39    This function is used to decode the `type_int` from the directory block of an OPUS file into a tuple of integers.
 40    Each integer in the tuple provides information about the associated data block.
 41
 42    Args:
 43        type_int: 32-bit integer decoded from file directory block
 44
 45    Returns:
 46        **block_type (tuple):** six-integer tuple which specifies the block type
 47    '''
 48    type_bit_str = format(type_int, '#034b')  # binary representation as string
 49    block_type = (
 50        int(type_bit_str[-2:], 2),
 51        int(type_bit_str[-4:-2], 2),
 52        int(type_bit_str[-10:-4], 2),
 53        int(type_bit_str[-17:-10], 2),
 54        int(type_bit_str[-19:-17], 2),
 55        int(type_bit_str[-22:-19], 2)
 56    )
 57    return block_type
 58
 59
 60def decode_str(size: int, blockbytes: bytes, offset: int) -> str:
 61    '''Decode string that is packed as bytes in `blockbytes` starting from `offset`.
 62
 63    Strings are frequently stored in OPUS files with a size designation that is larger than the actual string. The end
 64    of the string is designated by a terminator byte: b'\x00'. This function unpacks the string using the size
 65    designator, truncates at the terminator byte if found, and decodes as "latin-1"
 66
 67    Args:
 68        size: size (number of bytes) of the string
 69        blockbytes: raw bytes of an OPUS file block
 70        offset: offset location where string begins in blockbytes
 71
 72    Returns:
 73        string: decoded string
 74    '''
 75    fmt = '<' + str(size) + 's'
 76    try:
 77        val = struct.unpack_from(fmt, blockbytes, offset)[0]
 78        x00_pos = val.find(b'\x00')
 79        if x00_pos != -1:
 80            val = val[:x00_pos].decode('latin-1')
 81        else:
 82            val = val.decode('latin-1')
 83    except Exception as e:
 84        val = 'Failed to decode: ' + str(e)
 85    return val
 86
 87
 88def parse_header(filebytes: bytes) -> tuple:
 89    '''Parses the OPUS file header.
 90
 91    The header of an OPUS file contains some basic information about the file including the version number, location of
 92    the directory block, and number of blocks in the file. This header is first to be parsed as it specifies how to
 93    read the file directory block (which contains information about each block in the file)
 94
 95    Args:
 96        filebytes: raw bytes of OPUS file (all bytes)
 97
 98    Returns:
 99        **header_info (tuple):**  
100            (  
101                **version (float64):** program version number as a floating-point date (later versions always greater)  
102                **directory_start (int32):** pointer to start location of directory block (number of bytes)  
103                **max_blocks (int32):** maximum number of blocks supported by the directory block (this should only be
104                    relevant when trying to edit an OPUS file, i.e. when adding data blocks to a file)  
105                **num_blocks (int32):** total number of blocks in the opus file  
106            )
107    '''
108    version = struct.unpack_from('<d', filebytes, 4)[0]
109    directory_start = struct.unpack_from('<i', filebytes, 12)[0]
110    max_blocks = struct.unpack_from('<i', filebytes, 16)[0]
111    num_blocks = struct.unpack_from('<i', filebytes, 20)[0]
112    return version, directory_start, max_blocks, num_blocks
113
114
115def parse_directory(blockbytes: bytes) -> list:
116    '''Parses directory block of OPUS file and returns a list of block info tuples: (type, size, start).
117
118    The directory block of an OPUS file contains information about every block in the file. The block information is
119    stored as three int32 values: `type_int`, `size_int`, `start`.  `type_int` is an integer representation of the block
120    type. The bits of this `type_int` have meaning and are parsed into a tuple using `get_block_type`. The `size_int` is
121    the size of the block in 32-bit words. `start` is the starting location of the block (in number of bytes).
122
123    Args:
124        blockbytes: raw bytes of an OPUS file directory block
125
126    Returns:
127        **blocks (list):** list of block_info tuples
128            **block_info (tuple):**
129                (  
130                    **block_type (tuple):** six-integer tuple which specifies the block type (see: `get_block_type`)  
131                    **size (int):** size (number of bytes) of the block  
132                    **start (int):** pointer to start location of the block (number of bytes)
133                )
134    '''
135    loc = 0
136    blocks = []
137    while loc < len(blockbytes):
138        type_int, size_int, start = struct.unpack_from('<3i', blockbytes, loc)
139        loc = loc + 12
140        if start > 0:
141            block_type = get_block_type(type_int)
142            size = size_int*4
143            blocks.append((block_type, size, start))
144        else:
145            break
146    return blocks
147
148
149def parse_params(blockbytes: bytes) -> dict:
150    '''Parses the bytes in a parameter block and returns a dict containing the decoded keys and vals.
151
152    Parameter blocks are in the form: `XXX`, `dtype_code`, `size`, `val`.  `XXX` is a three char abbreviation of the
153    parameter (key). The value of the parameter is decoded according to the `dtype_code` and size integers to be either:
154    `int`, `float`, or `string`.
155
156    Args:
157        blockbytes: raw bytes of an OPUS file parameter block
158
159    Returns:
160        **items (tuple):** (key, value) pairs where key is three char string (lowercase) and value can be `int`, `float`
161            or `string`.
162    '''
163    loc = 0
164    params = dict()
165    while loc < len(blockbytes):
166        key = blockbytes[loc:loc + 3].decode('utf-8')
167        if key == 'END':
168            break
169        dtype_code, val_size = struct.unpack_from('<2h', blockbytes[loc + 4:loc + 8])
170        val_size = val_size * 2
171        if dtype_code == 0:
172            val = struct.unpack_from('<i', blockbytes, loc + 8)[0]
173        elif dtype_code == 1:
174            val = struct.unpack_from('<d', blockbytes, loc + 8)[0]
175        else:
176            val = decode_str(val_size, blockbytes, loc + 8)
177        params[key.lower()] = val
178        loc = loc + val_size + 8
179    return params
180
181
182def get_dpf_dtype_count(dpf: int, size: int) -> tuple:
183    '''Returns numpy dtype and array count from the data point format (dpf) and block size (in bytes).
184
185    Args:
186        dpf: data point format integer stored in data status block.
187            dpf = 1 -> array of float32
188            dpf = 2 -> array of int32
189        size: Block size in bytes.
190
191    Returns:
192        **dtype (numpy.dtype):** `numpy` dtype for defining an `ndarray` to store the data
193        **count (int):** length of array calculated from the block size and byte size of the dtype.
194    '''
195    if dpf == 2:
196        dtype = np.int32
197        count = round(size/4)
198    else:
199        dtype = np.float32
200        count = round(size/4)
201    return dtype, count
202
203
204def parse_data(blockbytes: bytes, dpf: int = 1) -> np.ndarray:
205    '''Parses the bytes in a data block and returns a `numpy` array.
206
207    Data blocks contain no metadata, only the y-values of a data array. Data arrays include: single-channel sample,
208    reference, phase, interferograms, and a variety of resultant data (transmission, absorption, etc.).  Every data
209    block should have a corresponding data status parameter block which can be used to generate the x-array values for
210    the data block. The data status block also specifies the data type of the data array with the `DPF` parameter. It
211    appears that OPUS currently exclusively stores data blocks as 32-bit floats, but has a reservation for 32-bit
212    integers when `DPF` = 2.
213
214    Args:
215        blockbytes: raw bytes of data block
216        dpf: data-point-format integer stored in corresponding data status block.
217
218    Returns:
219        **y_array (numpy.ndarray):** `numpy` array of y values contained in the data block
220    '''
221    dtype, count = get_dpf_dtype_count(dpf=dpf, size=len(blockbytes))
222    return np.frombuffer(blockbytes, dtype=dtype, count=count)
223
224
225def parse_data_compact(blockbytes: bytes, npt: int, dpf: int = 1) -> np.ndarray:
226    '''Parses the bytes in a data compact block and returns a `numpy` array.
227
228    Some data blocks are stored in the "Compact" block format that includes some metadata that preceeds the raw data. At
229    this time, the metadata is ignored, and the compact spectra is extracted from the last bytes of the block that fit
230    the known array size (specified with "npt" in corresponding data status block).
231
232    Args:
233        blockbytes: raw bytes of data block
234        npt: number of data points in the spectra (from data status block)
235        dpf: data-point-format integer stored in corresponding data status block.
236
237    Returns:
238        **y_array (numpy.ndarray):** `numpy` array of y values contained in the data block
239    '''
240    dtype, count = get_dpf_dtype_count(dpf=dpf, size=len(blockbytes))
241    return np.frombuffer(blockbytes, dtype=dtype, count=count)
242
243
244def parse_data_series(blockbytes: bytes, dpf: int = 1) -> dict:
245    '''Parses the bytes in a 3D data block (series of spectra) and returns a data `dict` containing data and metadata.
246
247    3D data blocks are structured differently than standard data blocks. In addition to the series of spectra, they
248    include metadata for each of the spectrum.  This function returns a `dict` containing all the extracted information
249    from the data block.  The series spectra is formed into a 2D array while metadata captured for each spectra is
250    formed into a 1D array (length = number of spectral measurements in the series).
251
252    Args:
253        blockbytes: raw bytes of the data series block
254        dpf: data-point-format integer stored in corresponding data status block.
255
256    Returns:
257        **data_dict (dict):** `dict` containing all extracted information from the data block  
258            {  
259                **version:** file format version number (should be 0)  
260                **num_blocks:** number of sub blocks; each sub block features a data spectra and associated metadata  
261                **offset:** offset in bytes to the first sub data block  
262                **data_size:** size in bytes of each sub data block  
263                **info_size:** size in bytes of the metadata info block immediately following the sub data block  
264                **store_table:** run numbers of the first and last blocks to keep track of skipped spectra  
265                **y:** 2D `numpy` array containing all spectra (C-order)  
266                **metadata arrays:** series of metadata arrays in 1D array format (e.g. `npt`, `mny`, `mxy`, `ert`).
267                    The most useful one is generally `ert`, which can be used as the time axis for 3D data plots.
268            }
269    '''
270    header = struct.unpack_from('<6i', blockbytes, 0)
271    data = {
272        'version': header[0],
273        'num_blocks': header[1],
274        'offset': header[2],
275        'data_size': header[3],
276        'info_size': header[4],
277    }
278    data['store_table'] = [struct.unpack_from('<2i', blockbytes, 24 + i * 8) for i in range(header[5])]
279    dtype, count = get_dpf_dtype_count(dpf, data['data_size'])
280    data['y'] = np.zeros((data['num_blocks'], count), dtype=dtype)
281    for entry in STRUCT_3D_INFO_BLOCK:
282        data[entry['key']] = np.zeros((data['num_blocks']), dtype=entry['dtype'])
283    offset = data['offset']
284    for i in range(data['num_blocks']):
285        data['y'][i] = np.frombuffer(blockbytes[offset:], dtype=dtype, count=count)
286        offset = offset + data['data_size']
287        info_vals = struct.unpack_from('<' + ''.join([e['fmt'] for e in STRUCT_3D_INFO_BLOCK]), blockbytes, offset)
288        for j, entry in enumerate(STRUCT_3D_INFO_BLOCK):
289            data[entry['key']][i] = info_vals[j]
290        offset = offset + data['info_size']
291    return data
292
293
294def parse_text(block_bytes: bytes, encoding='utf-8', backup_encoding='utf-16') -> str:
295    '''Parses and OPUS file block as text (e.g. history or file-log block).
296
297    The history (aka file-log) block of an OPUS file contains some information about how the file was generated and
298    edits that have been performed on the file.  This function parses the text block but does not take any steps to
299    parameterizing what is contained in the text.  The history block is generally not needed to retrieve the file data
300    and metadata, but might be useful for inspecting the file.
301
302    Args:
303        blockbytes: raw bytes of the text block (e.g. history or file-log)
304        encoding: string represting text encoding type. Can be set to "utf-16" for chinese character support
305        backup_encoding: if default encoding fails, this encoding string will be attempted.
306
307    Returns:
308        text: string of text contained in the file block.
309    '''
310    byte_string = struct.unpack('<' + str(len(block_bytes)) + 's', block_bytes)[0]
311    byte_strings = byte_string.split(b'\x00')
312    strings = []
313    for entry in byte_strings:
314        if entry != b'':
315            try:
316                strings.append(entry.decode(encoding))
317            except Exception:
318                try:
319                    strings.append(entry.decode(backup_encoding))
320                except Exception:
321                    try:
322                        strings.append(entry.decode('latin-1'))
323                    except Exception as e:
324                        strings.append('<Decode Exception>: ' + str(e))
325    return '\n'.join(strings)
326
327
328def parse_subreport(subreport_bytes: bytes) -> dict:
329    '''Parses the bytes of a subreport and returns the extracted data as a dictionary
330
331    Subreports are contained within a report block (e.g. Multi-Evaluation Test Report). A report can contain multiple
332    subreports, and they generally follow a table format. This sub-block is organized with a mini parameter block
333    followed by packed data. The mini parameter block contains information about how to read the packed data:
334        nco: number of columns
335        nln: number of rows
336        siz: size of mini parameter block (number of bytes)
337        src: size in bytes of entire row of data (offset for extracting column data from row 2, 3 ...)
338        f00, f01 ... fxx: start position of data in column 0, 1 ... xx (relative to end of mini param block)
339        t00, t01 ... txx: integer representing type of data (e.g. int32, float32, float64, str, etc.)
340        s00, s01 ... sxx: column header label
341        p00, p01 ... pxx: formatting string for numbers in column 0, 1 ... xx (not included for every column)
342
343    Args:
344        subreport_bytes: raw bytes of the subreport. Needs to start precisely where subreport begins, but can include
345        data beyond the end of the subreport (i.e. end of subreport does not need to be determined a priori).
346
347    Returns:
348        **subreport (dict):** `dict` containing subreport data and extraction/formatting parameters  
349            {  
350                **info:** `dict` of parameters extracted directly from subreport that describes how to read the data
351                    table and provides some basic metadata about the table (e.g. column header labels).
352                **data:** `list` of lists of data (table format) contained in the subreport
353            }
354    '''
355    info = parse_params(subreport_bytes)
356    data = []
357    for row in range(info['nln']):
358        data.append([])
359        for col in range(info['nco']):
360            offset = info['siz'] + row * info['src'] + info['f' + f'{col:02}']
361            type_int = info['t' + f'{col:02}']
362            if col < info['nco'] - 1:
363                size = min([type_int - 1000, info['f' + f'{col + 1:02}'] - info['f' + f'{col:02}']])
364            else:
365                size = info['src'] - info['f' + f'{col:02}']
366            if type_int > 1000:
367                val = decode_str(size, subreport_bytes, offset)
368            elif type_int in SUBREPORT_TYPE_FMT.keys():
369                fmt = SUBREPORT_TYPE_FMT[type_int]
370                val = struct.unpack_from(fmt, subreport_bytes, offset)[0]
371            else:
372                val = subreport_bytes[offset:offset + size]
373            data[row].append(val)
374    return {'info': info, 'data': data}
375
376
377def parse_report(blockbytes: bytes) -> dict:
378    '''Parses the report block of an OPUS file, such as Multi-Evaluation test reports, returning the report as a dict.
379
380    Report blocks are formatted in a very general way, potentially enabling a variety of different report structures.
381    This algorithm was developed using several OPUS files with a variety of different Multi-Evaluation Test Reports.
382    It is possible that other classes of test reports could be generated by OPUS that might change the structure, but
383    the overal organization and decoding methods should be similar.  In particular, the report block structure might
384    support multiple reports, but no such file has been available for testing to date.  This algorithm will extract a
385    single report and all the associated subreports.
386
387    Report blocks start with a mini parameter block that begins after the 12th byte.  It contains the following:
388        tit: Title of the report
389        f00: Starting position of the report summary data
390        Known unused parameters: bid, nrp, siz, e00, z00
391    This is followed by the report summary. For a multi-evaluation test report, this is a pair of tables summarizing the
392    methods applied to the spectrum.  It also specifies the number of subreports that follow, and the starting position
393    and title of each subreport. Some of the keys in this parameter set are described in the `parse_subreport` method.
394    Other parameters in the report summary include:
395        sub: Number of subreports
396        h00, h01 ... hxx: header labels of first summary table
397        v00, v01 ... vxx: corresponding values of first summary table
398        g00, g01 ... gxx: starting positions of each subreport relative to the start of this report summary
399        u00, u02 ... uxx: titles of each subreport
400    It should be noted that the only class of reports used for testing this algorithm were a variety of multi-evaluation
401    test reports. It is possible there are other similar report blocks OPUS writes that follow a similar structure but
402    could vary in some way that is incompatible with some of the assumptions made by this algorithm.
403
404    Args:
405        blockbytes: raw bytes of an OPUS file report block
406
407    Returns:
408        **report (dict):** `dict` containing report and subreport data 
409            {
410                **header:** `dict` of parameters from first mini param block
411                **info:** `dict` of parameters extracted from second mini param block
412                **data:** `list` of data that comprises second summary table
413                **subreports:** `list` of subreport `dict`s beneath the main report.
414            }
415    '''
416    header_ints = struct.unpack('<3i', blockbytes[:12])
417    header_info = parse_params(blockbytes[12:])
418    header_info['ints'] = header_ints
419    report_info = parse_subreport(blockbytes[header_info['f00']:])
420    report = {'header': header_info, 'info': report_info['info'], 'data': report_info['data']}
421    subreports = []
422    for idx in range(report_info['info']['sub']):
423        offset = header_info['f00'] + report_info['info']['g' + f'{idx:02}']
424        subreports.append(parse_subreport(blockbytes[offset:]))
425    report['subreports'] = subreports
426    return report