diff options
| -rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.cc | 128 | ||||
| -rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.h | 51 | ||||
| -rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.i | 6 | ||||
| -rw-r--r-- | gnuradio-core/src/python/gnuradio/parse_file_metadata.py | 29 | ||||
| -rw-r--r-- | gr-utils/src/python/gr_read_file_metadata | 33 | ||||
| -rw-r--r-- | grc/blocks/gr_file_meta_sink.xml | 8 |
6 files changed, 203 insertions, 52 deletions
diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.cc b/gnuradio-core/src/lib/io/gr_file_meta_sink.cc index aa34bf9b7..9d092d79f 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.cc +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.cc @@ -28,25 +28,31 @@ #include <gr_io_signature.h> #include <stdexcept> +#define HEADER_SIZE 117 + gr_file_meta_sink_sptr gr_make_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string &extra_dict) { return gnuradio::get_initial_sptr (new gr_file_meta_sink(itemsize, filename, - samp_rate, type, complex, + samp_rate, relative_rate, + type, complex, extra_dict)); } gr_file_meta_sink::gr_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string &extra_dict) : gr_sync_block("file_meta_sink", gr_make_io_signature(1, 1, itemsize), gr_make_io_signature(0, 0, 0)), gr_file_sink_base(filename, true), - d_itemsize(itemsize) + d_itemsize(itemsize), d_relative_rate(relative_rate), + d_total_seg_size(0) { if(!open(filename)) throw std::runtime_error("file_meta_sink: can't open file\n"); @@ -55,11 +61,11 @@ gr_file_meta_sink::gr_file_meta_sink(size_t itemsize, const char *filename, pmt_from_double(0)); d_header = pmt_make_dict(); - d_header = pmt_dict_add(d_header, mp("sr"), mp(samp_rate)); - d_header = pmt_dict_add(d_header, mp("time"), timestamp); + d_header = pmt_dict_add(d_header, mp("rx_rate"), mp(samp_rate)); + d_header = pmt_dict_add(d_header, mp("rx_time"), timestamp); d_header = pmt_dict_add(d_header, mp("type"), pmt_from_long(type)); d_header = pmt_dict_add(d_header, mp("cplx"), complex ? PMT_T : PMT_F); - d_header = pmt_dict_add(d_header, mp("strt"), pmt_from_uint64(109)); + d_header = pmt_dict_add(d_header, mp("strt"), pmt_from_uint64(HEADER_SIZE)); d_header = pmt_dict_add(d_header, mp("size"), pmt_from_uint64(0)); // handle extra dictionary @@ -72,6 +78,8 @@ gr_file_meta_sink::write_header(pmt_t header) do_update(); std::string header_str = pmt_serialize_str(header); + if(header_str.size() != HEADER_SIZE) + throw std::runtime_error("file_meta_sink: header is wrong size.\n"); size_t nwritten = 0; while(nwritten < header_str.size()) { @@ -80,21 +88,55 @@ gr_file_meta_sink::write_header(pmt_t header) nwritten += count; if((count == 0) && (ferror(d_fp))) { fclose(d_fp); - throw std::runtime_error("file_meta_sink: error writing header to file\n"); + throw std::runtime_error("file_meta_sink: error writing header to file.\n"); } } } -gr_file_meta_sink::~gr_file_meta_sink() +bool +gr_file_meta_sink::update_header(pmt_t key, pmt_t value) { - fseek(d_fp, 0, SEEK_SET); + // Special handling caveat to transform rate from radio source into + // the rate at this sink. + if(pmt_eq(key, mp("rx_rate"))) { + double rate = pmt_to_double(value); + value = pmt_from_double(rate*d_relative_rate); + } - // Replace the dictionary item with the data size now that we're - // done. - uint64_t s = nitems_read(0) * d_itemsize; - d_header = pmt_dict_delete(d_header, mp("size")); - d_header = pmt_dict_add(d_header, mp("size"), pmt_from_uint64(s)); - write_header(d_header); + if(pmt_dict_has_key(d_header, key)) { + d_header = pmt_dict_add(d_header, key, value); + return true; + } + else { + return false; + } +} + +uint64_t +gr_file_meta_sink::get_last_header_loc() +{ + uint64_t loc = 0; + pmt_t v = pmt_dict_ref(d_header, mp("strt"), PMT_NIL); + if(!pmt_eq(v, PMT_NIL)) + loc = pmt_to_uint64(v) - HEADER_SIZE; + return loc; +} + +gr_file_meta_sink::~gr_file_meta_sink() +{ + // Replace the last header block with the final count of the number + // of items. + uint64_t loc = get_last_header_loc(); + uint64_t seg_size = nitems_read(0) * d_itemsize - d_total_seg_size; + pmt_t s = pmt_from_uint64(seg_size); + //std::cerr << "Destructor" << std::endl; + //std::cerr << " location of last header: " << loc << std::endl; + //std::cerr << " nitems_read: " << nitems_read(0)*d_itemsize << std::endl; + //std::cerr << " Segment Size: " << seg_size << std::endl; + if(update_header(mp("size"), s)) { + fseek(d_fp, loc, SEEK_SET); + write_header(d_header); + } } int @@ -110,6 +152,60 @@ gr_file_meta_sink::work(int noutput_items, if(!d_fp) return noutput_items; // drop output on the floor + uint64_t abs_N = nitems_read(0); + uint64_t end_N = abs_N + (uint64_t)(noutput_items); + std::vector<gr_tag_t> all_tags; + get_tags_in_range(all_tags, 0, abs_N, end_N); + + std::vector<gr_tag_t>::iterator itr; + for(itr = all_tags.begin(); itr != all_tags.end(); itr++) { + // Special case where info is carried on the first tag, so we just + // overwrite the first header. + if(itr->offset == 0) { + if(update_header(itr->key, itr->value)) { + fseek(d_fp, 0, SEEK_SET); + write_header(d_header); + } + } + else { + // Update the last header info with the number of samples this + // block represents. + uint64_t loc = get_last_header_loc(); + uint64_t seg_size = itr->offset * d_itemsize - d_total_seg_size; + pmt_t s = pmt_from_uint64(seg_size); + //std::cerr << "Found Tag at: " << itr->offset*d_itemsize << std::endl; + //std::cerr << " last header starts at: " << loc << std::endl; + //std::cerr << " segment size is: " << seg_size << std::endl; + if(update_header(mp("size"), s)) { + fseek(d_fp, loc, SEEK_SET); + write_header(d_header); + } + + if(update_header(itr->key, itr->value)) { + // Otherwise, set current size of chunk to 0 and start of + // chunk based on current index + header size. + d_total_seg_size += seg_size; + s = pmt_from_uint64(0); + if(update_header(mp("size"), s)) { + // If we have multiple tags on the same offset, this makes + // sure we just overwrite the same header each time instead + // of creating a new header per tag. + uint64_t seg_start = loc; + if(seg_size != 0) + seg_start += HEADER_SIZE + seg_size; + pmt_t s = pmt_from_uint64(seg_start + HEADER_SIZE); + if(update_header(mp("strt"), s)) { + //std::cerr << "Adding new header" << std::endl; + //std::cerr << " new header start at: " << seg_start-HEADER_SIZE << std::endl; + //std::cerr << " new seg start at: " << seg_start << std::endl; + fseek(d_fp, seg_start, SEEK_SET); + write_header(d_header); + } + } + } + } + } + while(nwritten < noutput_items) { int count = fwrite(inbuf, d_itemsize, noutput_items - nwritten, d_fp); if(count == 0) // FIXME add error handling diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.h b/gnuradio-core/src/lib/io/gr_file_meta_sink.h index 741408de1..c0f30bdc0 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.h +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.h @@ -46,34 +46,71 @@ typedef boost::shared_ptr<gr_file_meta_sink> gr_file_meta_sink_sptr; GR_CORE_API gr_file_meta_sink_sptr gr_make_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string &extra_dict=""); /*! - * \brief Write stream to file. + * \brief Write stream to file with meta-data headers. * \ingroup sink_blk + * + * These files represent data as binary information in between + * meta-data headers. The headers contain information about the type + * of data and properties of the data in the next segment of + * samples. The information includes: + * + * rx_rate (double): sample rate of data. + * rx_time (uint64_t, double): time stamp of first sample in segment. + * type (gr_file_types as int32_t): data type. + * cplx (bool): Is data complex? + * strt (uint64_t): Starting byte of data in this segment. + * size (uint64_t): Size in bytes of data in this segment. + * + * Tags can be sent to the file to update the information, which will + * create a new header. Headers are found by searching from the first + * header (at position 0 in the file) and reading where the data + * segment starts plus the data segment size. Following will either be + * a new header or EOF. */ - class GR_CORE_API gr_file_meta_sink : public gr_sync_block, public gr_file_sink_base { - + /*! + * \brief Create a meta-data file sink. + * + * \param itemsize (size_t): Size of data type. + * \param filename (string): Name of file to write data to. + * \param samp_rate (double): Sample rate of data. If sample rate will be + * set by a tag, such as rx_tag from a UHD source, this is + * basically ignored. + * \param relative_rate (double): Rate chance from source of sample + * rate tag to sink. + * \param type (gr_file_types): Data type (int, float, etc.) + * \param complex (bool): If data stream is complex + * \param extra_dict (string): a serialized PMT dictionary of extra + * information. Currently not supported. + */ friend GR_CORE_API gr_file_meta_sink_sptr gr_make_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string &extra_dict); private: size_t d_itemsize; + double d_relative_rate; + uint64_t d_total_seg_size; pmt_t d_header; pmt_t d_extra_dict; protected: gr_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string &extra_dict); - void write_header(pmt_t header); + bool update_header(pmt_t key, pmt_t value); + uint64_t get_last_header_loc(); public: ~gr_file_meta_sink(); diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.i b/gnuradio-core/src/lib/io/gr_file_meta_sink.i index 27fb4debb..7d90ca304 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.i +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.i @@ -35,14 +35,16 @@ enum gr_file_types { gr_file_meta_sink_sptr gr_make_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string & extra_dict=""); class gr_file_meta_sink : public gr_sync_block, public gr_file_sink_base { protected: gr_file_meta_sink(size_t itemsize, const char *filename, - double samp_rate, gr_file_types type, bool complex, + double samp_rate, double relative_rate, + gr_file_types type, bool complex, const std::string & extra_dict); public: diff --git a/gnuradio-core/src/python/gnuradio/parse_file_metadata.py b/gnuradio-core/src/python/gnuradio/parse_file_metadata.py index 66cb4e447..cff7566e4 100644 --- a/gnuradio-core/src/python/gnuradio/parse_file_metadata.py +++ b/gnuradio-core/src/python/gnuradio/parse_file_metadata.py @@ -32,7 +32,7 @@ strt Start of data (or size of header) in bytes size Size of data in bytes ''' -HEADER_LENGTH = 109 +HEADER_LENGTH = 117 ftype_to_string = {gr.GR_FILE_BYTE: "bytes", gr.GR_FILE_SHORT: "short", gr.GR_FILE_INT: "int", @@ -49,7 +49,7 @@ ftype_to_size = {gr.GR_FILE_BYTE: gr.sizeof_char, gr.GR_FILE_FLOAT: gr.sizeof_float, gr.GR_FILE_DOUBLE: gr.sizeof_double} -def parse_header(p, VERBOSE=False): +def parse_header(p, hdr_start, VERBOSE=False): dump = gr.PMT_NIL info = dict() @@ -59,10 +59,10 @@ def parse_header(p, VERBOSE=False): sys.exit(1) # EXTRACT SAMPLE RATE - if(gr.pmt_dict_has_key(p, gr.pmt_string_to_symbol("sr"))): - r = gr.pmt_dict_ref(p, gr.pmt_string_to_symbol("sr"), dump) + if(gr.pmt_dict_has_key(p, gr.pmt_string_to_symbol("rx_rate"))): + r = gr.pmt_dict_ref(p, gr.pmt_string_to_symbol("rx_rate"), dump) samp_rate = gr.pmt_to_double(r) - info["sr"] = samp_rate + info["rx_rate"] = samp_rate if(VERBOSE): print "Sample Rate: {0} sps".format(samp_rate) else: @@ -70,14 +70,14 @@ def parse_header(p, VERBOSE=False): sys.exit(1) # EXTRACT TIME STAMP - if(gr.pmt_dict_has_key(p, gr.pmt_string_to_symbol("time"))): - r = gr.pmt_dict_ref(p, gr.pmt_string_to_symbol("time"), dump) + if(gr.pmt_dict_has_key(p, gr.pmt_string_to_symbol("rx_time"))): + r = gr.pmt_dict_ref(p, gr.pmt_string_to_symbol("rx_time"), dump) pmt_secs = gr.pmt_tuple_ref(r, 0) pmt_fracs = gr.pmt_tuple_ref(r, 1) secs = float(gr.pmt_to_uint64(pmt_secs)) fracs = gr.pmt_to_double(pmt_fracs) - t = secs + fracs/(1e9) - info["time"] = t + t = secs + fracs + info["rx_time"] = t if(VERBOSE): print "Seconds: {0}".format(t) else: @@ -107,14 +107,15 @@ def parse_header(p, VERBOSE=False): sys.stderr.write("Could not find key 'cplx': invalid or corrupt data file.\n") sys.exit(1) - # EXTRACT HEADER LENGTH + # EXTRACT WHERE CURRENT SEGMENT STARTS if(gr.pmt_dict_has_key(p, gr.pmt_string_to_symbol("strt"))): r = gr.pmt_dict_ref(p, gr.pmt_string_to_symbol("strt"), dump) - hdr_len = gr.pmt_to_uint64(r) - info["strt"] = hdr_len + seg_start = gr.pmt_to_uint64(r) + info["strt"] = seg_start if(VERBOSE): - print "Header Length: {0} bytes".format(hdr_len) - print "Extra Header? {0}".format(hdr_len > HEADER_LENGTH) + print "Segment Start: {0} bytes".format(seg_start) + print "Header Length: {0}".format((seg_start-hdr_start)) + print "Extra Header? {0}".format((seg_start-hdr_start) > HEADER_LENGTH) else: sys.stderr.write("Could not find key 'strt': invalid or corrupt data file.\n") sys.exit(1) diff --git a/gr-utils/src/python/gr_read_file_metadata b/gr-utils/src/python/gr_read_file_metadata index 657ad7c2b..47109aead 100644 --- a/gr-utils/src/python/gr_read_file_metadata +++ b/gr-utils/src/python/gr_read_file_metadata @@ -29,19 +29,28 @@ from gnuradio import parse_file_metadata def main(filename): handle = open(filename, "rb") - # just read out header bytes - header_str = handle.read(parse_file_metadata.HEADER_LENGTH) - - # Convert from string to PMT (should be a dictionary) - try: - header = gr.pmt_deserialize_str(header_str) - except RuntimeError: - sys.stderr.write("Could not deserialize header: invalid or corrupt data file.\n") - sys.exit(1) - #gr.pmt_print(header) + nread = 0 + while(True): + # read out next header bytes + hdr_start = handle.tell() + header_str = handle.read(parse_file_metadata.HEADER_LENGTH) + if(len(header_str) == 0): + break + + # Convert from string to PMT (should be a dictionary) + try: + header = gr.pmt_deserialize_str(header_str) + except RuntimeError: + sys.stderr.write("Could not deserialize header: invalid or corrupt data file.\n") + sys.exit(1) + #gr.pmt_print(header) + + info = parse_file_metadata.parse_header(header, hdr_start, True) + print "\n\n" + + nread += info['nbytes'] + parse_file_metadata.HEADER_LENGTH - info = parse_file_metadata.parse_header(header, True) - #print info + handle.seek(nread, 0) if __name__ == "__main__": usage="%prog: [options] filename" diff --git a/grc/blocks/gr_file_meta_sink.xml b/grc/blocks/gr_file_meta_sink.xml index 13a7030b0..e6f5f2bd6 100644 --- a/grc/blocks/gr_file_meta_sink.xml +++ b/grc/blocks/gr_file_meta_sink.xml @@ -8,7 +8,7 @@ <name>File Meta Sink</name> <key>gr_file_meta_sink</key> <import>from gnuradio import gr</import> - <make>gr.file_meta_sink($type.size*$vlen, $file, $samp_rate, $type.dtype, $type.cplx) + <make>gr.file_meta_sink($type.size*$vlen, $file, $samp_rate, $rel_rate, $type.dtype, $type.cplx) self.$(id).set_unbuffered($unbuffered)</make> <callback>set_unbuffered($unbuffered)</callback> <callback>open($file)</callback> @@ -65,6 +65,12 @@ self.$(id).set_unbuffered($unbuffered)</make> <type>real</type> </param> <param> + <name>Relative Rate Change</name> + <key>rel_rate</key> + <value>1</value> + <type>real</type> + </param> + <param> <name>Vec Length</name> <key>vlen</key> <value>1</value> |