diff options
author | Tom Rondeau | 2012-12-14 13:57:29 -0500 |
---|---|---|
committer | Tom Rondeau | 2012-12-14 13:57:29 -0500 |
commit | 3910c6da3c62232e6593e5fcfe53f5ece933a294 (patch) | |
tree | d3cc80bae32ced851c83f8393d5fea3b57020464 | |
parent | 75b01cc4f55c38dfe5bb329c23df591e43cf4c66 (diff) | |
download | gnuradio-3910c6da3c62232e6593e5fcfe53f5ece933a294.tar.gz gnuradio-3910c6da3c62232e6593e5fcfe53f5ece933a294.tar.bz2 gnuradio-3910c6da3c62232e6593e5fcfe53f5ece933a294.zip |
core: adding itemsize key to metadata header to allow for vectorized items.
This also simplifies some code in the source since we're told exactly what the items size is and don't have to infer.
Also adds an example using vector items.
-rw-r--r-- | docs/doxygen/other/metadata.dox | 9 | ||||
-rw-r--r-- | gnuradio-core/src/examples/metadata/file_metadata_vector_sink.grc | 219 | ||||
-rw-r--r-- | gnuradio-core/src/examples/metadata/file_metadata_vector_source.grc | 338 | ||||
-rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.cc | 9 | ||||
-rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.h | 2 | ||||
-rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_sink.i | 2 | ||||
-rw-r--r-- | gnuradio-core/src/lib/io/gr_file_meta_source.cc | 33 | ||||
-rw-r--r-- | gnuradio-core/src/python/gnuradio/parse_file_metadata.py | 21 | ||||
-rw-r--r-- | grc/blocks/gr_file_meta_source.xml | 8 |
9 files changed, 596 insertions, 45 deletions
diff --git a/docs/doxygen/other/metadata.dox b/docs/doxygen/other/metadata.dox index 5b20b9e73..4bc310531 100644 --- a/docs/doxygen/other/metadata.dox +++ b/docs/doxygen/other/metadata.dox @@ -16,9 +16,9 @@ Metadata files have headers that carry information about a segment of data within the file. The header structure is described in detail in the next section. A metadata file always starts with a header that describes the basic structure of the data. It contains information -about the data type, if it's complex, the sample rate of the segment, -the time stamp of the first sample of the segment, and information -regarding the header size and segment size. +about the item size, data type, if it's complex, the sample rate of +the segment, the time stamp of the first sample of the segment, and +information regarding the header size and segment size. Headers have two main tags associated with them: @@ -190,10 +190,11 @@ must be the same length and structure. As of now, we only have version - version: (char) version number (usually set to METADATA_VERSION) - rx_rate: (double) Stream's sample rate - rx_time: (pmt_t pair - (uint64_t, double)) Time stamp (format from UHD) +- size: (int) item size in bytes - reflects vector length if any. - type: (int) data type (enum below) - cplx: (bool) true if data is complex - strt: (uint64_t) start of data relative to current header -- size: (uint64_t) size of following data segment +- bytes: (uint64_t) size of following data segment in bytes The data types are indicated by an integer value from the following enumeration type: diff --git a/gnuradio-core/src/examples/metadata/file_metadata_vector_sink.grc b/gnuradio-core/src/examples/metadata/file_metadata_vector_sink.grc new file mode 100644 index 000000000..d67af3a66 --- /dev/null +++ b/gnuradio-core/src/examples/metadata/file_metadata_vector_sink.grc @@ -0,0 +1,219 @@ +<?xml version='1.0' encoding='ASCII'?> +<flow_graph> + <timestamp>Fri Dec 14 13:52:31 2012</timestamp> + <block> + <key>variable</key> + <param> + <key>id</key> + <value>samp_rate</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>value</key> + <value>200000</value> + </param> + <param> + <key>_coordinate</key> + <value>(208, 11)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_vector_source_x</key> + <param> + <key>id</key> + <value>gr_vector_source_x_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>vector</key> + <value>10*[0,1,2,3,4,5,6,7,8,9]</value> + </param> + <param> + <key>repeat</key> + <value>True</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>_coordinate</key> + <value>(67, 100)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_file_meta_sink</key> + <param> + <key>id</key> + <value>gr_file_meta_sink_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>file</key> + <value>/tmp/metadat_file.out</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>samp_rate</key> + <value>samp_rate</value> + </param> + <param> + <key>rel_rate</key> + <value>1</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>max_seg_size</key> + <value>1000000</value> + </param> + <param> + <key>extra_dict</key> + <value></value> + </param> + <param> + <key>detached</key> + <value>False</value> + </param> + <param> + <key>unbuffered</key> + <value>False</value> + </param> + <param> + <key>_coordinate</key> + <value>(559, 60)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>options</key> + <param> + <key>id</key> + <value>file_metadata_vector_sink</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>title</key> + <value></value> + </param> + <param> + <key>author</key> + <value></value> + </param> + <param> + <key>description</key> + <value></value> + </param> + <param> + <key>window_size</key> + <value>1280, 1024</value> + </param> + <param> + <key>generate_options</key> + <value>no_gui</value> + </param> + <param> + <key>category</key> + <value>Custom</value> + </param> + <param> + <key>run_options</key> + <value>run</value> + </param> + <param> + <key>run</key> + <value>True</value> + </param> + <param> + <key>max_nouts</key> + <value>0</value> + </param> + <param> + <key>realtime_scheduling</key> + <value></value> + </param> + <param> + <key>_coordinate</key> + <value>(10, 10)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_head</key> + <param> + <key>id</key> + <value>gr_head_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>num_items</key> + <value>10010000</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>_coordinate</key> + <value>(325, 108)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <connection> + <source_block_id>gr_vector_source_x_0</source_block_id> + <sink_block_id>gr_head_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> + <connection> + <source_block_id>gr_head_0</source_block_id> + <sink_block_id>gr_file_meta_sink_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> +</flow_graph> diff --git a/gnuradio-core/src/examples/metadata/file_metadata_vector_source.grc b/gnuradio-core/src/examples/metadata/file_metadata_vector_source.grc new file mode 100644 index 000000000..0662865f1 --- /dev/null +++ b/gnuradio-core/src/examples/metadata/file_metadata_vector_source.grc @@ -0,0 +1,338 @@ +<?xml version='1.0' encoding='ASCII'?> +<flow_graph> + <timestamp>Fri Dec 14 13:55:43 2012</timestamp> + <block> + <key>options</key> + <param> + <key>id</key> + <value>file_metadata_source</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>title</key> + <value></value> + </param> + <param> + <key>author</key> + <value></value> + </param> + <param> + <key>description</key> + <value></value> + </param> + <param> + <key>window_size</key> + <value>1280, 1024</value> + </param> + <param> + <key>generate_options</key> + <value>qt_gui</value> + </param> + <param> + <key>category</key> + <value>Custom</value> + </param> + <param> + <key>run_options</key> + <value>prompt</value> + </param> + <param> + <key>run</key> + <value>True</value> + </param> + <param> + <key>max_nouts</key> + <value>0</value> + </param> + <param> + <key>realtime_scheduling</key> + <value></value> + </param> + <param> + <key>_coordinate</key> + <value>(10, 10)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>variable</key> + <param> + <key>id</key> + <value>samp_rate</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>value</key> + <value>200000</value> + </param> + <param> + <key>_coordinate</key> + <value>(208, 11)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_file_sink</key> + <param> + <key>id</key> + <value>gr_file_sink_1</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>file</key> + <value>/tmp/received_data.out</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>unbuffered</key> + <value>False</value> + </param> + <param> + <key>_coordinate</key> + <value>(564, 215)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_tag_debug</key> + <param> + <key>id</key> + <value>gr_tag_debug_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>name</key> + <value>Test</value> + </param> + <param> + <key>num_inputs</key> + <value>1</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>display</key> + <value>True</value> + </param> + <param> + <key>_coordinate</key> + <value>(563, 298)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_file_meta_source</key> + <param> + <key>id</key> + <value>gr_file_meta_source_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>file</key> + <value>/tmp/metadat_file.out</value> + </param> + <param> + <key>repeat</key> + <value>False</value> + </param> + <param> + <key>detached</key> + <value>False</value> + </param> + <param> + <key>hdr_file</key> + <value>/tmp/metadat_file.out.hdr</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>_coordinate</key> + <value>(51, 199)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>qtgui_time_sink_x</key> + <param> + <key>id</key> + <value>qtgui_time_sink_x_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>name</key> + <value>QT GUI Plot</value> + </param> + <param> + <key>size</key> + <value>1024</value> + </param> + <param> + <key>bw</key> + <value>samp_rate</value> + </param> + <param> + <key>nconnections</key> + <value>1</value> + </param> + <param> + <key>gui_hint</key> + <value></value> + </param> + <param> + <key>_coordinate</key> + <value>(746, 116)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>gr_throttle</key> + <param> + <key>id</key> + <value>gr_throttle_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>samples_per_second</key> + <value>samp_rate</value> + </param> + <param> + <key>vlen</key> + <value>10</value> + </param> + <param> + <key>_coordinate</key> + <value>(322, 223)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <block> + <key>blocks_vector_to_stream</key> + <param> + <key>id</key> + <value>blocks_vector_to_stream_0</value> + </param> + <param> + <key>_enabled</key> + <value>True</value> + </param> + <param> + <key>type</key> + <value>complex</value> + </param> + <param> + <key>num_items</key> + <value>10</value> + </param> + <param> + <key>vlen</key> + <value>1</value> + </param> + <param> + <key>_coordinate</key> + <value>(526, 132)</value> + </param> + <param> + <key>_rotation</key> + <value>0</value> + </param> + </block> + <connection> + <source_block_id>gr_throttle_0</source_block_id> + <sink_block_id>gr_file_sink_1</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> + <connection> + <source_block_id>gr_throttle_0</source_block_id> + <sink_block_id>gr_tag_debug_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> + <connection> + <source_block_id>gr_file_meta_source_0</source_block_id> + <sink_block_id>gr_throttle_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> + <connection> + <source_block_id>blocks_vector_to_stream_0</source_block_id> + <sink_block_id>qtgui_time_sink_x_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> + <connection> + <source_block_id>gr_throttle_0</source_block_id> + <sink_block_id>blocks_vector_to_stream_0</sink_block_id> + <source_key>0</source_key> + <sink_key>0</sink_key> + </connection> +</flow_graph> diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.cc b/gnuradio-core/src/lib/io/gr_file_meta_sink.cc index b016dd1d3..ab0acbdb4 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.cc +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.cc @@ -116,10 +116,11 @@ gr_file_meta_sink::gr_file_meta_sink(size_t itemsize, const std::string &filenam d_header = pmt_dict_add(d_header, mp("version"), mp(METADATA_VERSION)); d_header = pmt_dict_add(d_header, mp("rx_rate"), mp(samp_rate)); d_header = pmt_dict_add(d_header, mp("rx_time"), timestamp); + d_header = pmt_dict_add(d_header, mp("size"), pmt_from_long(d_itemsize)); d_header = pmt_dict_add(d_header, mp("type"), pmt_from_long(type)); d_header = pmt_dict_add(d_header, mp("cplx"), complex ? PMT_T : PMT_F); d_header = pmt_dict_add(d_header, mp("strt"), pmt_from_uint64(METADATA_HEADER_SIZE+d_extra_size)); - d_header = pmt_dict_add(d_header, mp("size"), pmt_from_uint64(0)); + d_header = pmt_dict_add(d_header, mp("bytes"), pmt_from_uint64(0)); do_update(); @@ -305,7 +306,7 @@ gr_file_meta_sink::update_last_header_inline() size_t hdrlen = pmt_to_uint64(pmt_dict_ref(d_header, mp("strt"), PMT_NIL)); size_t seg_size = d_itemsize*d_total_seg_size; pmt_t s = pmt_from_uint64(seg_size); - update_header(mp("size"), s); + update_header(mp("bytes"), s); update_header(mp("strt"), pmt_from_uint64(METADATA_HEADER_SIZE+d_extra_size)); fseek(d_fp, -seg_size-hdrlen, SEEK_CUR); write_header(d_fp, d_header, d_extra); @@ -320,7 +321,7 @@ gr_file_meta_sink::update_last_header_detached() size_t hdrlen = pmt_to_uint64(pmt_dict_ref(d_header, mp("strt"), PMT_NIL)); size_t seg_size = d_itemsize*d_total_seg_size; pmt_t s = pmt_from_uint64(seg_size); - update_header(mp("size"), s); + update_header(mp("bytes"), s); update_header(mp("strt"), pmt_from_uint64(METADATA_HEADER_SIZE+d_extra_size)); fseek(d_hdr_fp, -hdrlen, SEEK_CUR); write_header(d_hdr_fp, d_header, d_extra); @@ -333,7 +334,7 @@ gr_file_meta_sink::write_and_update() // based on current index + header size. //uint64_t loc = get_last_header_loc(); pmt_t s = pmt_from_uint64(0); - update_header(mp("size"), s); + update_header(mp("bytes"), s); // If we have multiple tags on the same offset, this makes // sure we just overwrite the same header each time instead diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.h b/gnuradio-core/src/lib/io/gr_file_meta_sink.h index c0219ac25..9b67cc4c8 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.h +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.h @@ -31,7 +31,7 @@ using namespace pmt; const char METADATA_VERSION = 0; -const size_t METADATA_HEADER_SIZE = 134; +const size_t METADATA_HEADER_SIZE = 149; enum gr_file_types { GR_FILE_BYTE=0, diff --git a/gnuradio-core/src/lib/io/gr_file_meta_sink.i b/gnuradio-core/src/lib/io/gr_file_meta_sink.i index 743e89730..6fa34913b 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_sink.i +++ b/gnuradio-core/src/lib/io/gr_file_meta_sink.i @@ -23,7 +23,7 @@ GR_SWIG_BLOCK_MAGIC(gr,file_meta_sink) const char METADATA_VERSION = 0; -const size_t METADATA_HEADER_SIZE = 134; +const size_t METADATA_HEADER_SIZE = 149; enum gr_file_types { GR_FILE_BYTE=0, diff --git a/gnuradio-core/src/lib/io/gr_file_meta_source.cc b/gnuradio-core/src/lib/io/gr_file_meta_source.cc index 669c5d477..d940e5edc 100644 --- a/gnuradio-core/src/lib/io/gr_file_meta_source.cc +++ b/gnuradio-core/src/lib/io/gr_file_meta_source.cc @@ -228,38 +228,17 @@ gr_file_meta_source::parse_header(pmt_t hdr, uint64_t offset, throw std::runtime_error("file_meta_source: Could not extract time stamp.\n"); } - // GET DATA TYPE - if(pmt_dict_has_key(hdr, pmt_string_to_symbol("type"))) { - int t = pmt_to_long(pmt_dict_ref(hdr, pmt_string_to_symbol("type"), PMT_NIL)); - switch(t) { - case(GR_FILE_CHAR): d_itemsize = sizeof(char); break; - case(GR_FILE_SHORT): d_itemsize = sizeof(short); break; - case(GR_FILE_INT): d_itemsize = sizeof(int); break; - case(GR_FILE_LONG): d_itemsize = sizeof(long int); break; - case(GR_FILE_LONG_LONG): d_itemsize = sizeof(long long int); break; - case(GR_FILE_FLOAT): d_itemsize = sizeof(float); break; - case(GR_FILE_DOUBLE): d_itemsize = sizeof(double); break; - default: - throw std::runtime_error("file_meta_source: Could not determine data type size.\n"); - } - } - else { - throw std::runtime_error("file_meta_source: Could not extract data type.\n"); - } - - // GET COMPLEX INDICATOR - if(pmt_dict_has_key(hdr, pmt_string_to_symbol("cplx"))) { - bool cplx = pmt_to_bool(pmt_dict_ref(hdr, pmt_string_to_symbol("cplx"), PMT_NIL)); - if(cplx) - d_itemsize *= 2; + // GET ITEM SIZE OF DATA + if(pmt_dict_has_key(hdr, pmt_string_to_symbol("size"))) { + d_itemsize = pmt_to_long(pmt_dict_ref(hdr, pmt_string_to_symbol("size"), PMT_NIL)); } else { - throw std::runtime_error("file_meta_source: Could not extract complex indicator.\n"); + throw std::runtime_error("file_meta_source: Could not extract item size.\n"); } // GET SEGMENT SIZE - if(pmt_dict_has_key(hdr, pmt_string_to_symbol("size"))) { - d_seg_size = pmt_to_uint64(pmt_dict_ref(hdr, pmt_string_to_symbol("size"), PMT_NIL)); + if(pmt_dict_has_key(hdr, pmt_string_to_symbol("bytes"))) { + d_seg_size = pmt_to_uint64(pmt_dict_ref(hdr, pmt_string_to_symbol("bytes"), PMT_NIL)); // Convert from bytes to items d_seg_size /= d_itemsize; diff --git a/gnuradio-core/src/python/gnuradio/parse_file_metadata.py b/gnuradio-core/src/python/gnuradio/parse_file_metadata.py index fa54db746..0cee5a02c 100644 --- a/gnuradio-core/src/python/gnuradio/parse_file_metadata.py +++ b/gnuradio-core/src/python/gnuradio/parse_file_metadata.py @@ -96,6 +96,17 @@ def parse_header(p, VERBOSE=False): sys.stderr.write("Could not find key 'time': invalid or corrupt data file.\n") sys.exit(1) + # EXTRACT ITEM SIZE + if(pmt.pmt_dict_has_key(p, pmt.pmt_string_to_symbol("size"))): + r = pmt.pmt_dict_ref(p, pmt.pmt_string_to_symbol("size"), dump) + dsize = pmt.pmt_to_long(r) + info["size"] = dsize + if(VERBOSE): + print "Item size: {0}".format(dsize) + else: + sys.stderr.write("Could not find key 'size': invalid or corrupt data file.\n") + sys.exit(1) + # EXTRACT DATA TYPE if(pmt.pmt_dict_has_key(p, pmt.pmt_string_to_symbol("type"))): r = pmt.pmt_dict_ref(p, pmt.pmt_string_to_symbol("type"), dump) @@ -135,15 +146,11 @@ def parse_header(p, VERBOSE=False): sys.exit(1) # EXTRACT SIZE OF DATA - if(pmt.pmt_dict_has_key(p, pmt.pmt_string_to_symbol("size"))): - r = pmt.pmt_dict_ref(p, pmt.pmt_string_to_symbol("size"), dump) + if(pmt.pmt_dict_has_key(p, pmt.pmt_string_to_symbol("bytes"))): + r = pmt.pmt_dict_ref(p, pmt.pmt_string_to_symbol("bytes"), dump) nbytes = pmt.pmt_to_uint64(r) - # Multiply itemsize by 2 if complex - if(cplx): mult=2 - else: mult=1 - - nitems = nbytes/(ftype_to_size[dtype]*mult) + nitems = nbytes/dsize info["nitems"] = nitems info["nbytes"] = nbytes diff --git a/grc/blocks/gr_file_meta_source.xml b/grc/blocks/gr_file_meta_source.xml index 9e5fcb0a2..8f667961e 100644 --- a/grc/blocks/gr_file_meta_source.xml +++ b/grc/blocks/gr_file_meta_source.xml @@ -80,9 +80,15 @@ <value></value> <type>file_open</type> </param> + <param> + <name>Vec Length</name> + <key>vlen</key> + <value>1</value> + <type>int</type> + </param> <source> <name>out</name> <type>$type</type> - <vlen>1</vlen> + <vlen>$vlen</vlen> </source> </block> |