<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.8"/>
<title>SphinxBase: src/sphinx_lmtools/sphinx_lm_eval.c Source File</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="navtree.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="resize.js"></script>
<script type="text/javascript" src="navtree.js"></script>
<script type="text/javascript">
$(document).ready(initResizable);
$(window).load(resizeHeight);
</script>
<link href="doxygen.css" rel="stylesheet" type="text/css" />
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td style="padding-left: 0.5em;">
<div id="projectname">SphinxBase
 <span id="projectnumber">0.6</span>
</div>
</td>
</tr>
</tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.8 -->
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="index.html"><span>Main Page</span></a></li>
<li><a href="pages.html"><span>Related Pages</span></a></li>
<li><a href="annotated.html"><span>Data Structures</span></a></li>
<li class="current"><a href="files.html"><span>Files</span></a></li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="files.html"><span>File List</span></a></li>
<li><a href="globals.html"><span>Globals</span></a></li>
</ul>
</div>
</div><!-- top -->
<div id="side-nav" class="ui-resizable side-nav-resizable">
<div id="nav-tree">
<div id="nav-tree-contents">
<div id="nav-sync" class="sync"></div>
</div>
</div>
<div id="splitbar" style="-moz-user-select:none;"
class="ui-resizable-handle">
</div>
</div>
<script type="text/javascript">
$(document).ready(function(){initNavTree('sphinx__lm__eval_8c_source.html','');});
</script>
<div id="doc-content">
<div class="header">
<div class="headertitle">
<div class="title">sphinx_lm_eval.c</div> </div>
</div><!--header-->
<div class="contents">
<a href="sphinx__lm__eval_8c.html">Go to the documentation of this file.</a><div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */</span></div>
<div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment">/* ====================================================================</span></div>
<div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="comment"> * Copyright (c) 2008 Carnegie Mellon University. All rights </span></div>
<div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="comment"> * reserved.</span></div>
<div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="comment"> * Redistribution and use in source and binary forms, with or without</span></div>
<div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="comment"> * modification, are permitted provided that the following conditions</span></div>
<div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="comment"> * are met:</span></div>
<div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="comment"> * 1. Redistributions of source code must retain the above copyright</span></div>
<div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="comment"> * notice, this list of conditions and the following disclaimer. </span></div>
<div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="comment"> * 2. Redistributions in binary form must reproduce the above copyright</span></div>
<div class="line"><a name="l00014"></a><span class="lineno"> 14</span> <span class="comment"> * notice, this list of conditions and the following disclaimer in</span></div>
<div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="comment"> * the documentation and/or other materials provided with the</span></div>
<div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="comment"> * distribution.</span></div>
<div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="comment"> * This work was supported in part by funding from the Defense Advanced </span></div>
<div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="comment"> * Research Projects Agency and the National Science Foundation of the </span></div>
<div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="comment"> * United States of America, and the CMU Sphinx Speech Consortium.</span></div>
<div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00022"></a><span class="lineno"> 22</span> <span class="comment"> * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND </span></div>
<div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="comment"> * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, </span></div>
<div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="comment"> * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR</span></div>
<div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="comment"> * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY</span></div>
<div class="line"><a name="l00026"></a><span class="lineno"> 26</span> <span class="comment"> * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,</span></div>
<div class="line"><a name="l00027"></a><span class="lineno"> 27</span> <span class="comment"> * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT </span></div>
<div class="line"><a name="l00028"></a><span class="lineno"> 28</span> <span class="comment"> * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, </span></div>
<div class="line"><a name="l00029"></a><span class="lineno"> 29</span> <span class="comment"> * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY </span></div>
<div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="comment"> * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT </span></div>
<div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="comment"> * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE </span></div>
<div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="comment"> * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</span></div>
<div class="line"><a name="l00033"></a><span class="lineno"> 33</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00034"></a><span class="lineno"> 34</span> <span class="comment"> * ====================================================================</span></div>
<div class="line"><a name="l00035"></a><span class="lineno"> 35</span> <span class="comment"> *</span></div>
<div class="line"><a name="l00036"></a><span class="lineno"> 36</span> <span class="comment"> */</span></div>
<div class="line"><a name="l00041"></a><span class="lineno"> 41</span> <span class="preprocessor">#include <<a class="code" href="logmath_8h.html">sphinxbase/logmath.h</a>></span></div>
<div class="line"><a name="l00042"></a><span class="lineno"> 42</span> <span class="preprocessor">#include <<a class="code" href="ngram__model_8h.html">sphinxbase/ngram_model.h</a>></span></div>
<div class="line"><a name="l00043"></a><span class="lineno"> 43</span> <span class="preprocessor">#include <<a class="code" href="cmd__ln_8h.html">sphinxbase/cmd_ln.h</a>></span></div>
<div class="line"><a name="l00044"></a><span class="lineno"> 44</span> <span class="preprocessor">#include <<a class="code" href="ckd__alloc_8h.html">sphinxbase/ckd_alloc.h</a>></span></div>
<div class="line"><a name="l00045"></a><span class="lineno"> 45</span> <span class="preprocessor">#include <<a class="code" href="err_8h.html">sphinxbase/err.h</a>></span></div>
<div class="line"><a name="l00046"></a><span class="lineno"> 46</span> <span class="preprocessor">#include <<a class="code" href="pio_8h.html">sphinxbase/pio.h</a>></span></div>
<div class="line"><a name="l00047"></a><span class="lineno"> 47</span> <span class="preprocessor">#include <<a class="code" href="strfuncs_8h.html">sphinxbase/strfuncs.h</a>></span></div>
<div class="line"><a name="l00048"></a><span class="lineno"> 48</span> </div>
<div class="line"><a name="l00049"></a><span class="lineno"> 49</span> <span class="preprocessor">#include <stdio.h></span></div>
<div class="line"><a name="l00050"></a><span class="lineno"> 50</span> <span class="preprocessor">#include <string.h></span></div>
<div class="line"><a name="l00051"></a><span class="lineno"> 51</span> <span class="preprocessor">#include <math.h></span></div>
<div class="line"><a name="l00052"></a><span class="lineno"> 52</span> </div>
<div class="line"><a name="l00053"></a><span class="lineno"> 53</span> <span class="keyword">static</span> <span class="keyword">const</span> <a class="code" href="structarg__t.html">arg_t</a> defn[] = {</div>
<div class="line"><a name="l00054"></a><span class="lineno"> 54</span>  { <span class="stringliteral">"-help"</span>,</div>
<div class="line"><a name="l00055"></a><span class="lineno"> 55</span>  <a class="code" href="cmd__ln_8h.html#ac7d08ff59bb6905c3375162e75913e88">ARG_BOOLEAN</a>,</div>
<div class="line"><a name="l00056"></a><span class="lineno"> 56</span>  <span class="stringliteral">"no"</span>,</div>
<div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  <span class="stringliteral">"Shows the usage of the tool"</span>},</div>
<div class="line"><a name="l00058"></a><span class="lineno"> 58</span> </div>
<div class="line"><a name="l00059"></a><span class="lineno"> 59</span>  { <span class="stringliteral">"-logbase"</span>,</div>
<div class="line"><a name="l00060"></a><span class="lineno"> 60</span>  <a class="code" href="cmd__ln_8h.html#aceb617c8572cf5ad6257b35e6d8919e7">ARG_FLOAT64</a>,</div>
<div class="line"><a name="l00061"></a><span class="lineno"> 61</span>  <span class="stringliteral">"1.0001"</span>,</div>
<div class="line"><a name="l00062"></a><span class="lineno"> 62</span>  <span class="stringliteral">"Base in which all log-likelihoods calculated"</span> },</div>
<div class="line"><a name="l00063"></a><span class="lineno"> 63</span> </div>
<div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  { <span class="stringliteral">"-lm"</span>,</div>
<div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00066"></a><span class="lineno"> 66</span>  NULL,</div>
<div class="line"><a name="l00067"></a><span class="lineno"> 67</span>  <span class="stringliteral">"Language model file"</span>},</div>
<div class="line"><a name="l00068"></a><span class="lineno"> 68</span> </div>
<div class="line"><a name="l00069"></a><span class="lineno"> 69</span>  { <span class="stringliteral">"-probdef"</span>,</div>
<div class="line"><a name="l00070"></a><span class="lineno"> 70</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00071"></a><span class="lineno"> 71</span>  NULL,</div>
<div class="line"><a name="l00072"></a><span class="lineno"> 72</span>  <span class="stringliteral">"Probability definition file for classes in LM"</span>},</div>
<div class="line"><a name="l00073"></a><span class="lineno"> 73</span> </div>
<div class="line"><a name="l00074"></a><span class="lineno"> 74</span>  { <span class="stringliteral">"-lmctlfn"</span>,</div>
<div class="line"><a name="l00075"></a><span class="lineno"> 75</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00076"></a><span class="lineno"> 76</span>  NULL,</div>
<div class="line"><a name="l00077"></a><span class="lineno"> 77</span>  <span class="stringliteral">"Control file listing a set of language models"</span>},</div>
<div class="line"><a name="l00078"></a><span class="lineno"> 78</span> </div>
<div class="line"><a name="l00079"></a><span class="lineno"> 79</span>  { <span class="stringliteral">"-lmname"</span>,</div>
<div class="line"><a name="l00080"></a><span class="lineno"> 80</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00081"></a><span class="lineno"> 81</span>  NULL,</div>
<div class="line"><a name="l00082"></a><span class="lineno"> 82</span>  <span class="stringliteral">"Name of language model in -lmctlfn to use for all utterances"</span> },</div>
<div class="line"><a name="l00083"></a><span class="lineno"> 83</span> </div>
<div class="line"><a name="l00084"></a><span class="lineno"> 84</span>  { <span class="stringliteral">"-lsn"</span>,</div>
<div class="line"><a name="l00085"></a><span class="lineno"> 85</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00086"></a><span class="lineno"> 86</span>  NULL,</div>
<div class="line"><a name="l00087"></a><span class="lineno"> 87</span>  <span class="stringliteral">"Transcription file to evaluate"</span>},</div>
<div class="line"><a name="l00088"></a><span class="lineno"> 88</span> </div>
<div class="line"><a name="l00089"></a><span class="lineno"> 89</span>  { <span class="stringliteral">"-text"</span>,</div>
<div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  <a class="code" href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a>,</div>
<div class="line"><a name="l00091"></a><span class="lineno"> 91</span>  <span class="stringliteral">"Text string to evaluate"</span>},</div>
<div class="line"><a name="l00092"></a><span class="lineno"> 92</span> </div>
<div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  { <span class="stringliteral">"-mmap"</span>,</div>
<div class="line"><a name="l00094"></a><span class="lineno"> 94</span>  <a class="code" href="cmd__ln_8h.html#ac7d08ff59bb6905c3375162e75913e88">ARG_BOOLEAN</a>,</div>
<div class="line"><a name="l00095"></a><span class="lineno"> 95</span>  <span class="stringliteral">"no"</span>,</div>
<div class="line"><a name="l00096"></a><span class="lineno"> 96</span>  <span class="stringliteral">"Use memory-mapped I/O for reading binary LM files"</span>},</div>
<div class="line"><a name="l00097"></a><span class="lineno"> 97</span> </div>
<div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  { <span class="stringliteral">"-lw"</span>,</div>
<div class="line"><a name="l00099"></a><span class="lineno"> 99</span>  <a class="code" href="cmd__ln_8h.html#ad9b1952e4f1def9ee6a88791375b3901">ARG_FLOAT32</a>,</div>
<div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <span class="stringliteral">"1.0"</span>,</div>
<div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  <span class="stringliteral">"Language model weight"</span> },</div>
<div class="line"><a name="l00102"></a><span class="lineno"> 102</span> </div>
<div class="line"><a name="l00103"></a><span class="lineno"> 103</span>  { <span class="stringliteral">"-wip"</span>,</div>
<div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  <a class="code" href="cmd__ln_8h.html#ad9b1952e4f1def9ee6a88791375b3901">ARG_FLOAT32</a>,</div>
<div class="line"><a name="l00105"></a><span class="lineno"> 105</span>  <span class="stringliteral">"1.0"</span>,</div>
<div class="line"><a name="l00106"></a><span class="lineno"> 106</span>  <span class="stringliteral">"Word insertion probability"</span> },</div>
<div class="line"><a name="l00107"></a><span class="lineno"> 107</span> </div>
<div class="line"><a name="l00108"></a><span class="lineno"> 108</span>  { <span class="stringliteral">"-uw"</span>,</div>
<div class="line"><a name="l00109"></a><span class="lineno"> 109</span>  <a class="code" href="cmd__ln_8h.html#ad9b1952e4f1def9ee6a88791375b3901">ARG_FLOAT32</a>,</div>
<div class="line"><a name="l00110"></a><span class="lineno"> 110</span>  <span class="stringliteral">"1.0"</span>,</div>
<div class="line"><a name="l00111"></a><span class="lineno"> 111</span>  <span class="stringliteral">"Unigram probability weight (interpolated with uniform distribution)"</span>},</div>
<div class="line"><a name="l00112"></a><span class="lineno"> 112</span> </div>
<div class="line"><a name="l00113"></a><span class="lineno"> 113</span>  { <span class="stringliteral">"-verbose"</span>,</div>
<div class="line"><a name="l00114"></a><span class="lineno"> 114</span>  <a class="code" href="cmd__ln_8h.html#ac7d08ff59bb6905c3375162e75913e88">ARG_BOOLEAN</a>,</div>
<div class="line"><a name="l00115"></a><span class="lineno"> 115</span>  <span class="stringliteral">"no"</span>,</div>
<div class="line"><a name="l00116"></a><span class="lineno"> 116</span>  <span class="stringliteral">"Print details of perplexity calculation"</span> },</div>
<div class="line"><a name="l00117"></a><span class="lineno"> 117</span> </div>
<div class="line"><a name="l00118"></a><span class="lineno"> 118</span>  <span class="comment">/* FIXME: Support -lmstartsym, -lmendsym, -lmctlfn, -ctl_lm */</span></div>
<div class="line"><a name="l00119"></a><span class="lineno"> 119</span>  { NULL, 0, NULL, NULL }</div>
<div class="line"><a name="l00120"></a><span class="lineno"> 120</span> };</div>
<div class="line"><a name="l00121"></a><span class="lineno"> 121</span> </div>
<div class="line"><a name="l00122"></a><span class="lineno"> 122</span> <span class="keyword">static</span> <span class="keywordtype">int</span> verbose;</div>
<div class="line"><a name="l00123"></a><span class="lineno"> 123</span> </div>
<div class="line"><a name="l00124"></a><span class="lineno"> 124</span> <span class="keyword">static</span> <span class="keywordtype">int</span></div>
<div class="line"><a name="l00125"></a><span class="lineno"> 125</span> calc_entropy(<a class="code" href="structngram__model__s.html">ngram_model_t</a> *lm, <span class="keywordtype">char</span> **words, int32 n,</div>
<div class="line"><a name="l00126"></a><span class="lineno"> 126</span>  int32 *out_n_ccs, int32 *out_n_oovs, int32 *out_lm_score)</div>
<div class="line"><a name="l00127"></a><span class="lineno"> 127</span> {</div>
<div class="line"><a name="l00128"></a><span class="lineno"> 128</span>  int32 *wids;</div>
<div class="line"><a name="l00129"></a><span class="lineno"> 129</span>  int32 startwid;</div>
<div class="line"><a name="l00130"></a><span class="lineno"> 130</span>  int32 i, ch, nccs, noovs, unk;</div>
<div class="line"><a name="l00131"></a><span class="lineno"> 131</span> </div>
<div class="line"><a name="l00132"></a><span class="lineno"> 132</span>  <span class="keywordflow">if</span> (n == 0)</div>
<div class="line"><a name="l00133"></a><span class="lineno"> 133</span>  <span class="keywordflow">return</span> 0;</div>
<div class="line"><a name="l00134"></a><span class="lineno"> 134</span> </div>
<div class="line"><a name="l00135"></a><span class="lineno"> 135</span>  unk = <a class="code" href="ngram__model_8h.html#a1469e9e1c8516a77c9ac1e248a61ef4e">ngram_unknown_wid</a>(lm);</div>
<div class="line"><a name="l00136"></a><span class="lineno"> 136</span> </div>
<div class="line"><a name="l00137"></a><span class="lineno"> 137</span>  <span class="comment">/* Reverse this array into an array of word IDs. */</span></div>
<div class="line"><a name="l00138"></a><span class="lineno"> 138</span>  wids = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e">ckd_calloc</a>(n, <span class="keyword">sizeof</span>(*wids));</div>
<div class="line"><a name="l00139"></a><span class="lineno"> 139</span>  <span class="keywordflow">for</span> (i = 0; i < n; ++i)</div>
<div class="line"><a name="l00140"></a><span class="lineno"> 140</span>  wids[n-i-1] = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880">ngram_wid</a>(lm, words[i]);</div>
<div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  <span class="comment">/* Skip <s> as it's a context cue (HACK, this should be configurable). */</span></div>
<div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  startwid = <a class="code" href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880">ngram_wid</a>(lm, <span class="stringliteral">"<s>"</span>);</div>
<div class="line"><a name="l00143"></a><span class="lineno"> 143</span> </div>
<div class="line"><a name="l00144"></a><span class="lineno"> 144</span>  <span class="comment">/* Now evaluate the list of words in reverse using the</span></div>
<div class="line"><a name="l00145"></a><span class="lineno"> 145</span> <span class="comment"> * remainder of the array as the history. */</span></div>
<div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  ch = noovs = nccs = 0;</div>
<div class="line"><a name="l00147"></a><span class="lineno"> 147</span>  <span class="keywordflow">for</span> (i = 0; i < n; ++i) {</div>
<div class="line"><a name="l00148"></a><span class="lineno"> 148</span>  int32 n_used;</div>
<div class="line"><a name="l00149"></a><span class="lineno"> 149</span>  int32 prob;</div>
<div class="line"><a name="l00150"></a><span class="lineno"> 150</span> </div>
<div class="line"><a name="l00151"></a><span class="lineno"> 151</span>  <span class="comment">/* Skip <s> as it's a context cue (HACK, this should be configurable). */</span></div>
<div class="line"><a name="l00152"></a><span class="lineno"> 152</span>  <span class="keywordflow">if</span> (wids[i] == startwid) {</div>
<div class="line"><a name="l00153"></a><span class="lineno"> 153</span>  ++nccs;</div>
<div class="line"><a name="l00154"></a><span class="lineno"> 154</span>  <span class="keywordflow">continue</span>;</div>
<div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  }</div>
<div class="line"><a name="l00156"></a><span class="lineno"> 156</span>  <span class="comment">/* Skip and count OOVs. */</span></div>
<div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <span class="keywordflow">if</span> (wids[i] == <a class="code" href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab">NGRAM_INVALID_WID</a> || wids[i] == unk) {</div>
<div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  ++noovs;</div>
<div class="line"><a name="l00159"></a><span class="lineno"> 159</span>  <span class="keywordflow">continue</span>;</div>
<div class="line"><a name="l00160"></a><span class="lineno"> 160</span>  }</div>
<div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <span class="comment">/* Sum up information for each N-gram */</span></div>
<div class="line"><a name="l00162"></a><span class="lineno"> 162</span>  prob = <a class="code" href="ngram__model_8h.html#a6ac5799e78ea4ad82a11e2439016471e">ngram_ng_score</a>(lm,</div>
<div class="line"><a name="l00163"></a><span class="lineno"> 163</span>  wids[i], wids + i + 1,</div>
<div class="line"><a name="l00164"></a><span class="lineno"> 164</span>  n - i - 1, &n_used);</div>
<div class="line"><a name="l00165"></a><span class="lineno"> 165</span>  <span class="keywordflow">if</span> (verbose) {</div>
<div class="line"><a name="l00166"></a><span class="lineno"> 166</span>  <span class="keywordtype">int</span> m;</div>
<div class="line"><a name="l00167"></a><span class="lineno"> 167</span>  printf(<span class="stringliteral">"log P(%s|"</span>, <a class="code" href="ngram__model_8h.html#a96e36290a005c03464ea6c637ccde2f5">ngram_word</a>(lm, wids[i]));</div>
<div class="line"><a name="l00168"></a><span class="lineno"> 168</span>  m = i + <a class="code" href="ngram__model_8h.html#a462d374099a4fe8b3c3195b5e2013545">ngram_model_get_size</a>(lm) - 1;</div>
<div class="line"><a name="l00169"></a><span class="lineno"> 169</span>  <span class="keywordflow">if</span> (m >= n)</div>
<div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  m = n - 1;</div>
<div class="line"><a name="l00171"></a><span class="lineno"> 171</span>  <span class="keywordflow">while</span> (m > i) {</div>
<div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  printf(<span class="stringliteral">"%s "</span>, <a class="code" href="ngram__model_8h.html#a96e36290a005c03464ea6c637ccde2f5">ngram_word</a>(lm, wids[m--]));</div>
<div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  }</div>
<div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  printf(<span class="stringliteral">") = %d\n"</span>, prob);</div>
<div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  }</div>
<div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  ch -= prob;</div>
<div class="line"><a name="l00177"></a><span class="lineno"> 177</span>  }</div>
<div class="line"><a name="l00178"></a><span class="lineno"> 178</span> </div>
<div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  <span class="keywordflow">if</span> (out_n_ccs) *out_n_ccs = nccs;</div>
<div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  <span class="keywordflow">if</span> (out_n_oovs) *out_n_oovs = noovs;</div>
<div class="line"><a name="l00181"></a><span class="lineno"> 181</span> </div>
<div class="line"><a name="l00182"></a><span class="lineno"> 182</span>  <span class="comment">/* Calculate cross-entropy CH = - 1/N sum log P(W|H) */</span></div>
<div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  n -= (nccs + noovs);</div>
<div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  <span class="keywordflow">if</span> (n <= 0)</div>
<div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  <span class="keywordflow">return</span> 0;</div>
<div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  <span class="keywordflow">if</span> (out_lm_score)</div>
<div class="line"><a name="l00187"></a><span class="lineno"> 187</span>  *out_lm_score = -ch;</div>
<div class="line"><a name="l00188"></a><span class="lineno"> 188</span>  <span class="keywordflow">return</span> ch / n;</div>
<div class="line"><a name="l00189"></a><span class="lineno"> 189</span> }</div>
<div class="line"><a name="l00190"></a><span class="lineno"> 190</span> </div>
<div class="line"><a name="l00191"></a><span class="lineno"> 191</span> <span class="keyword">static</span> <span class="keywordtype">void</span></div>
<div class="line"><a name="l00192"></a><span class="lineno"> 192</span> evaluate_file(<a class="code" href="structngram__model__s.html">ngram_model_t</a> *lm, <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath, <span class="keyword">const</span> <span class="keywordtype">char</span> *lsnfn)</div>
<div class="line"><a name="l00193"></a><span class="lineno"> 193</span> {</div>
<div class="line"><a name="l00194"></a><span class="lineno"> 194</span>  FILE *fh;</div>
<div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  <a class="code" href="structlineiter__t.html">lineiter_t</a> *litor;</div>
<div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  int32 nccs, noovs, nwords, lscr;</div>
<div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  float64 ch, log_to_log2;;</div>
<div class="line"><a name="l00198"></a><span class="lineno"> 198</span> </div>
<div class="line"><a name="l00199"></a><span class="lineno"> 199</span>  <span class="keywordflow">if</span> ((fh = fopen(lsnfn, <span class="stringliteral">"r"</span>)) == NULL)</div>
<div class="line"><a name="l00200"></a><span class="lineno"> 200</span>  <a class="code" href="err_8h.html#a5229a1d58f5f5e69963a8d038ff5bc3e">E_FATAL_SYSTEM</a>(<span class="stringliteral">"failed to open transcript file %s"</span>, lsnfn);</div>
<div class="line"><a name="l00201"></a><span class="lineno"> 201</span> </div>
<div class="line"><a name="l00202"></a><span class="lineno"> 202</span>  <span class="comment">/* We have to keep ch in floating-point to avoid overflows, so</span></div>
<div class="line"><a name="l00203"></a><span class="lineno"> 203</span> <span class="comment"> * we might as well use log2. */</span></div>
<div class="line"><a name="l00204"></a><span class="lineno"> 204</span>  log_to_log2 = log(<a class="code" href="logmath_8h.html#a6114206ec0321d7015c42fc7b81cb83e">logmath_get_base</a>(lmath)) / log(2);</div>
<div class="line"><a name="l00205"></a><span class="lineno"> 205</span>  nccs = noovs = nwords = 0;</div>
<div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  ch = 0.0;</div>
<div class="line"><a name="l00207"></a><span class="lineno"> 207</span>  <span class="keywordflow">for</span> (litor = <a class="code" href="pio_8h.html#a22d0125ab198f02f8bbe543417d99566">lineiter_start</a>(fh); litor; litor = <a class="code" href="pio_8h.html#aff8df0b6928746d61b3520555263f71e">lineiter_next</a>(litor)) {</div>
<div class="line"><a name="l00208"></a><span class="lineno"> 208</span>  <span class="keywordtype">char</span> **words;</div>
<div class="line"><a name="l00209"></a><span class="lineno"> 209</span>  int32 n, tmp_ch, tmp_noovs, tmp_nccs, tmp_lscr;</div>
<div class="line"><a name="l00210"></a><span class="lineno"> 210</span> </div>
<div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  n = <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f">str2words</a>(litor->buf, NULL, 0);</div>
<div class="line"><a name="l00212"></a><span class="lineno"> 212</span>  <span class="keywordflow">if</span> (n < 0)</div>
<div class="line"><a name="l00213"></a><span class="lineno"> 213</span>  <a class="code" href="err_8h.html#a1a4495946ab2449d61108fe829a94613">E_FATAL</a>(<span class="stringliteral">"str2words(line, NULL, 0) = %d, should not happen\n"</span>, n);</div>
<div class="line"><a name="l00214"></a><span class="lineno"> 214</span>  <span class="keywordflow">if</span> (n == 0) <span class="comment">/* Do nothing! */</span></div>
<div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  <span class="keywordflow">continue</span>;</div>
<div class="line"><a name="l00216"></a><span class="lineno"> 216</span>  words = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e">ckd_calloc</a>(n, <span class="keyword">sizeof</span>(*words));</div>
<div class="line"><a name="l00217"></a><span class="lineno"> 217</span>  <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f">str2words</a>(litor->buf, words, n);</div>
<div class="line"><a name="l00218"></a><span class="lineno"> 218</span> </div>
<div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  <span class="comment">/* Remove any utterance ID (FIXME: has to be a single "word") */</span></div>
<div class="line"><a name="l00220"></a><span class="lineno"> 220</span>  <span class="keywordflow">if</span> (words[n-1][0] == <span class="charliteral">'('</span></div>
<div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  && words[n-1][strlen(words[n-1])-1] == <span class="charliteral">')'</span>)</div>
<div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  n = n - 1;</div>
<div class="line"><a name="l00223"></a><span class="lineno"> 223</span> </div>
<div class="line"><a name="l00224"></a><span class="lineno"> 224</span>  tmp_ch = calc_entropy(lm, words, n, &tmp_nccs,</div>
<div class="line"><a name="l00225"></a><span class="lineno"> 225</span>  &tmp_noovs, &tmp_lscr);</div>
<div class="line"><a name="l00226"></a><span class="lineno"> 226</span> </div>
<div class="line"><a name="l00227"></a><span class="lineno"> 227</span>  ch += (float64) tmp_ch * (n - tmp_nccs - tmp_noovs) * log_to_log2;</div>
<div class="line"><a name="l00228"></a><span class="lineno"> 228</span>  nccs += tmp_nccs;</div>
<div class="line"><a name="l00229"></a><span class="lineno"> 229</span>  noovs += tmp_noovs;</div>
<div class="line"><a name="l00230"></a><span class="lineno"> 230</span>  lscr += tmp_lscr;</div>
<div class="line"><a name="l00231"></a><span class="lineno"> 231</span>  nwords += n;</div>
<div class="line"><a name="l00232"></a><span class="lineno"> 232</span>  </div>
<div class="line"><a name="l00233"></a><span class="lineno"> 233</span>  <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf">ckd_free</a>(words);</div>
<div class="line"><a name="l00234"></a><span class="lineno"> 234</span>  }</div>
<div class="line"><a name="l00235"></a><span class="lineno"> 235</span> </div>
<div class="line"><a name="l00236"></a><span class="lineno"> 236</span>  ch /= (nwords - nccs - noovs);</div>
<div class="line"><a name="l00237"></a><span class="lineno"> 237</span>  printf(<span class="stringliteral">"cross-entropy: %f bits\n"</span>, ch);</div>
<div class="line"><a name="l00238"></a><span class="lineno"> 238</span> </div>
<div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  <span class="comment">/* Calculate perplexity pplx = exp CH */</span></div>
<div class="line"><a name="l00240"></a><span class="lineno"> 240</span>  printf(<span class="stringliteral">"perplexity: %f\n"</span>, pow(2.0, ch));</div>
<div class="line"><a name="l00241"></a><span class="lineno"> 241</span>  printf(<span class="stringliteral">"lm score: %d\n"</span>, lscr);</div>
<div class="line"><a name="l00242"></a><span class="lineno"> 242</span> </div>
<div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  <span class="comment">/* Report OOVs and CCs */</span></div>
<div class="line"><a name="l00244"></a><span class="lineno"> 244</span>  printf(<span class="stringliteral">"%d words evaluated\n"</span>, nwords);</div>
<div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  printf(<span class="stringliteral">"%d OOVs (%.2f%%), %d context cues removed\n"</span>,</div>
<div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  noovs, (<span class="keywordtype">double</span>)noovs / nwords * 100, nccs);</div>
<div class="line"><a name="l00247"></a><span class="lineno"> 247</span> }</div>
<div class="line"><a name="l00248"></a><span class="lineno"> 248</span> </div>
<div class="line"><a name="l00249"></a><span class="lineno"> 249</span> <span class="keyword">static</span> <span class="keywordtype">void</span></div>
<div class="line"><a name="l00250"></a><span class="lineno"> 250</span> evaluate_string(<a class="code" href="structngram__model__s.html">ngram_model_t</a> *lm, <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath, <span class="keyword">const</span> <span class="keywordtype">char</span> *text)</div>
<div class="line"><a name="l00251"></a><span class="lineno"> 251</span> {</div>
<div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  <span class="keywordtype">char</span> *textfoo;</div>
<div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  <span class="keywordtype">char</span> **words;</div>
<div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  int32 n, ch, noovs, nccs, lscr;</div>
<div class="line"><a name="l00255"></a><span class="lineno"> 255</span> </div>
<div class="line"><a name="l00256"></a><span class="lineno"> 256</span>  <span class="comment">/* Split it into an array of strings. */</span></div>
<div class="line"><a name="l00257"></a><span class="lineno"> 257</span>  textfoo = <a class="code" href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393">ckd_salloc</a>(text);</div>
<div class="line"><a name="l00258"></a><span class="lineno"> 258</span>  n = <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f">str2words</a>(textfoo, NULL, 0);</div>
<div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="keywordflow">if</span> (n < 0)</div>
<div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  <a class="code" href="err_8h.html#a1a4495946ab2449d61108fe829a94613">E_FATAL</a>(<span class="stringliteral">"str2words(textfoo, NULL, 0) = %d, should not happen\n"</span>, n);</div>
<div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  <span class="keywordflow">if</span> (n == 0) <span class="comment">/* Do nothing! */</span></div>
<div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  <span class="keywordflow">return</span>;</div>
<div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  words = <a class="code" href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e">ckd_calloc</a>(n, <span class="keyword">sizeof</span>(*words));</div>
<div class="line"><a name="l00264"></a><span class="lineno"> 264</span>  <a class="code" href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f">str2words</a>(textfoo, words, n);</div>
<div class="line"><a name="l00265"></a><span class="lineno"> 265</span> </div>
<div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  ch = calc_entropy(lm, words, n, &nccs, &noovs, &lscr);</div>
<div class="line"><a name="l00267"></a><span class="lineno"> 267</span> </div>
<div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  printf(<span class="stringliteral">"input: %s\n"</span>, text);</div>
<div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  printf(<span class="stringliteral">"cross-entropy: %f bits\n"</span>,</div>
<div class="line"><a name="l00270"></a><span class="lineno"> 270</span>  ch * log(<a class="code" href="logmath_8h.html#a6114206ec0321d7015c42fc7b81cb83e">logmath_get_base</a>(lmath)) / log(2));</div>
<div class="line"><a name="l00271"></a><span class="lineno"> 271</span> </div>
<div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  <span class="comment">/* Calculate perplexity pplx = exp CH */</span></div>
<div class="line"><a name="l00273"></a><span class="lineno"> 273</span>  printf(<span class="stringliteral">"perplexity: %f\n"</span>, <a class="code" href="logmath_8h.html#ae8b0a168e29e448c0d6de66dc46e099e">logmath_exp</a>(lmath, ch));</div>
<div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  printf(<span class="stringliteral">"lm score: %d\n"</span>, lscr);</div>
<div class="line"><a name="l00275"></a><span class="lineno"> 275</span> </div>
<div class="line"><a name="l00276"></a><span class="lineno"> 276</span>  <span class="comment">/* Report OOVs and CCs */</span></div>
<div class="line"><a name="l00277"></a><span class="lineno"> 277</span>  printf(<span class="stringliteral">"%d words evaluated\n"</span>, n);</div>
<div class="line"><a name="l00278"></a><span class="lineno"> 278</span>  printf(<span class="stringliteral">"%d OOVs, %d context cues removed\n"</span>,</div>
<div class="line"><a name="l00279"></a><span class="lineno"> 279</span>  noovs, nccs);</div>
<div class="line"><a name="l00280"></a><span class="lineno"> 280</span> </div>
<div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf">ckd_free</a>(textfoo);</div>
<div class="line"><a name="l00282"></a><span class="lineno"> 282</span>  <a class="code" href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf">ckd_free</a>(words);</div>
<div class="line"><a name="l00283"></a><span class="lineno"> 283</span> }</div>
<div class="line"><a name="l00284"></a><span class="lineno"> 284</span> </div>
<div class="line"><a name="l00285"></a><span class="lineno"> 285</span> <span class="keywordtype">int</span></div>
<div class="line"><a name="l00286"></a><span class="lineno"> 286</span> main(<span class="keywordtype">int</span> argc, <span class="keywordtype">char</span> *argv[])</div>
<div class="line"><a name="l00287"></a><span class="lineno"> 287</span> {</div>
<div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  <a class="code" href="structcmd__ln__t.html">cmd_ln_t</a> *<a class="code" href="structsphinx__wave2feat__s.html#a484d308befbde315664da8520ebc410d">config</a>;</div>
<div class="line"><a name="l00289"></a><span class="lineno"> 289</span>  <a class="code" href="structngram__model__s.html">ngram_model_t</a> *lm = NULL;</div>
<div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  <a class="code" href="structlogmath__s.html">logmath_t</a> *lmath;</div>
<div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  <span class="keyword">const</span> <span class="keywordtype">char</span> *lmfn, *probdefn, *lsnfn, *text;</div>
<div class="line"><a name="l00292"></a><span class="lineno"> 292</span> </div>
<div class="line"><a name="l00293"></a><span class="lineno"> 293</span>  <span class="keywordflow">if</span> ((config = <a class="code" href="cmd__ln_8h.html#aa5a3a9e49198d8fd0dd3424fb880b6b6">cmd_ln_parse_r</a>(NULL, defn, argc, argv, TRUE)) == NULL)</div>
<div class="line"><a name="l00294"></a><span class="lineno"> 294</span>  <span class="keywordflow">return</span> 1;</div>
<div class="line"><a name="l00295"></a><span class="lineno"> 295</span> </div>
<div class="line"><a name="l00296"></a><span class="lineno"> 296</span>  verbose = <a class="code" href="cmd__ln_8h.html#a159e691c95089689cf9a8f85a67830a6">cmd_ln_boolean_r</a>(config, <span class="stringliteral">"-verbose"</span>);</div>
<div class="line"><a name="l00297"></a><span class="lineno"> 297</span> </div>
<div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  <span class="comment">/* Create log math object. */</span></div>
<div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  <span class="keywordflow">if</span> ((lmath = <a class="code" href="logmath_8h.html#a5835860c5f6a703c80c0214f816f0b11">logmath_init</a></div>
<div class="line"><a name="l00300"></a><span class="lineno"> 300</span>  (cmd_ln_float64_r(config, <span class="stringliteral">"-logbase"</span>), 0, 0)) == NULL) {</div>
<div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  <a class="code" href="err_8h.html#a1a4495946ab2449d61108fe829a94613">E_FATAL</a>(<span class="stringliteral">"Failed to initialize log math\n"</span>);</div>
<div class="line"><a name="l00302"></a><span class="lineno"> 302</span>  }</div>
<div class="line"><a name="l00303"></a><span class="lineno"> 303</span> </div>
<div class="line"><a name="l00304"></a><span class="lineno"> 304</span>  <span class="comment">/* Load the language model. */</span></div>
<div class="line"><a name="l00305"></a><span class="lineno"> 305</span>  lmfn = <a class="code" href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a>(config, <span class="stringliteral">"-lm"</span>);</div>
<div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  <span class="keywordflow">if</span> (lmfn == NULL</div>
<div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  || (lm = <a class="code" href="ngram__model_8h.html#ab0c840f2bdfc38cea08bb70054f76624">ngram_model_read</a>(config, lmfn,</div>
<div class="line"><a name="l00308"></a><span class="lineno"> 308</span>  <a class="code" href="ngram__model_8h.html#a406c0d64c15a9d1749d07c8ab6e0ae74a441701bf8ae0a2b79716feb31b5f257a">NGRAM_AUTO</a>, lmath)) == NULL) {</div>
<div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  <a class="code" href="err_8h.html#a1a4495946ab2449d61108fe829a94613">E_FATAL</a>(<span class="stringliteral">"Failed to load language model from %s\n"</span>,</div>
<div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  <a class="code" href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a>(config, <span class="stringliteral">"-lm"</span>));</div>
<div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  }</div>
<div class="line"><a name="l00312"></a><span class="lineno"> 312</span>  <span class="keywordflow">if</span> ((probdefn = <a class="code" href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a>(config, <span class="stringliteral">"-probdef"</span>)) != NULL)</div>
<div class="line"><a name="l00313"></a><span class="lineno"> 313</span>  <a class="code" href="ngram__model_8h.html#a9b2a86c23543158754373c5456fe890d">ngram_model_read_classdef</a>(lm, probdefn);</div>
<div class="line"><a name="l00314"></a><span class="lineno"> 314</span>  <a class="code" href="ngram__model_8h.html#aa4b8d7c1f3d873b8458c0cfee13af4da">ngram_model_apply_weights</a>(lm,</div>
<div class="line"><a name="l00315"></a><span class="lineno"> 315</span>  cmd_ln_float32_r(config, <span class="stringliteral">"-lw"</span>),</div>
<div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  cmd_ln_float32_r(config, <span class="stringliteral">"-wip"</span>),</div>
<div class="line"><a name="l00317"></a><span class="lineno"> 317</span>  cmd_ln_float32_r(config, <span class="stringliteral">"-uw"</span>));</div>
<div class="line"><a name="l00318"></a><span class="lineno"> 318</span> </div>
<div class="line"><a name="l00319"></a><span class="lineno"> 319</span>  <span class="comment">/* Now evaluate some text. */</span></div>
<div class="line"><a name="l00320"></a><span class="lineno"> 320</span>  lsnfn = <a class="code" href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a>(config, <span class="stringliteral">"-lsn"</span>);</div>
<div class="line"><a name="l00321"></a><span class="lineno"> 321</span>  text = <a class="code" href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a>(config, <span class="stringliteral">"-text"</span>);</div>
<div class="line"><a name="l00322"></a><span class="lineno"> 322</span>  <span class="keywordflow">if</span> (lsnfn) {</div>
<div class="line"><a name="l00323"></a><span class="lineno"> 323</span>  evaluate_file(lm, lmath, lsnfn);</div>
<div class="line"><a name="l00324"></a><span class="lineno"> 324</span>  }</div>
<div class="line"><a name="l00325"></a><span class="lineno"> 325</span>  <span class="keywordflow">else</span> <span class="keywordflow">if</span> (text) {</div>
<div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  evaluate_string(lm, lmath, text);</div>
<div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  }</div>
<div class="line"><a name="l00328"></a><span class="lineno"> 328</span> </div>
<div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  <span class="keywordflow">return</span> 0;</div>
<div class="line"><a name="l00330"></a><span class="lineno"> 330</span> }</div>
<div class="ttc" id="cmd__ln_8h_html"><div class="ttname"><a href="cmd__ln_8h.html">cmd_ln.h</a></div><div class="ttdoc">Command-line and other configurationparsing and handling. </div></div>
<div class="ttc" id="ngram__model_8h_html_ab0c840f2bdfc38cea08bb70054f76624"><div class="ttname"><a href="ngram__model_8h.html#ab0c840f2bdfc38cea08bb70054f76624">ngram_model_read</a></div><div class="ttdeci">SPHINXBASE_EXPORT ngram_model_t * ngram_model_read(cmd_ln_t *config, const char *file_name, ngram_file_type_t file_type, logmath_t *lmath)</div><div class="ttdoc">Read an N-Gram model from a file on disk. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00121">ngram_model.c:121</a></div></div>
<div class="ttc" id="strfuncs_8h_html"><div class="ttname"><a href="strfuncs_8h.html">strfuncs.h</a></div><div class="ttdoc">Miscellaneous useful string functions. </div></div>
<div class="ttc" id="ckd__alloc_8h_html_aa00ef21903bc4f8a972488417adc8d2e"><div class="ttname"><a href="ckd__alloc_8h.html#aa00ef21903bc4f8a972488417adc8d2e">ckd_calloc</a></div><div class="ttdeci">#define ckd_calloc(n, sz)</div><div class="ttdoc">Macros to simplify the use of above functions. </div><div class="ttdef"><b>Definition:</b> <a href="ckd__alloc_8h_source.html#l00248">ckd_alloc.h:248</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a1469e9e1c8516a77c9ac1e248a61ef4e"><div class="ttname"><a href="ngram__model_8h.html#a1469e9e1c8516a77c9ac1e248a61ef4e">ngram_unknown_wid</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 ngram_unknown_wid(ngram_model_t *model)</div><div class="ttdoc">Get the unknown word ID for a language model. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00644">ngram_model.c:644</a></div></div>
<div class="ttc" id="ckd__alloc_8h_html"><div class="ttname"><a href="ckd__alloc_8h.html">ckd_alloc.h</a></div><div class="ttdoc">Sphinx's memory allocation/deallocation routines. </div></div>
<div class="ttc" id="ngram__model_8h_html_ad03d4355d4ea659815dc25bce8d83880"><div class="ttname"><a href="ngram__model_8h.html#ad03d4355d4ea659815dc25bce8d83880">ngram_wid</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 ngram_wid(ngram_model_t *model, const char *word)</div><div class="ttdoc">Look up numerical word ID. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00771">ngram_model.c:771</a></div></div>
<div class="ttc" id="structlogmath__s_html"><div class="ttname"><a href="structlogmath__s.html">logmath_s</a></div><div class="ttdef"><b>Definition:</b> <a href="logmath_8c_source.html#l00049">logmath.c:49</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a3d4b3dddd0ff67e13d30c9bf053d01ab"><div class="ttname"><a href="ngram__model_8h.html#a3d4b3dddd0ff67e13d30c9bf053d01ab">NGRAM_INVALID_WID</a></div><div class="ttdeci">#define NGRAM_INVALID_WID</div><div class="ttdoc">Impossible word ID. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8h_source.html#l00084">ngram_model.h:84</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_aa5a3a9e49198d8fd0dd3424fb880b6b6"><div class="ttname"><a href="cmd__ln_8h.html#aa5a3a9e49198d8fd0dd3424fb880b6b6">cmd_ln_parse_r</a></div><div class="ttdeci">SPHINXBASE_EXPORT cmd_ln_t * cmd_ln_parse_r(cmd_ln_t *inout_cmdln, arg_t const *defn, int32 argc, char *argv[], int32 strict)</div><div class="ttdoc">Parse a list of strings into argumetns. </div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8c_source.html#l00551">cmd_ln.c:551</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_a4de5ed5fcf59a18b24bc9f6449cc9356"><div class="ttname"><a href="cmd__ln_8h.html#a4de5ed5fcf59a18b24bc9f6449cc9356">ARG_STRING</a></div><div class="ttdeci">#define ARG_STRING</div><div class="ttdoc">String argument (optional). </div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8h_source.html#l00114">cmd_ln.h:114</a></div></div>
<div class="ttc" id="structlineiter__t_html"><div class="ttname"><a href="structlineiter__t.html">lineiter_t</a></div><div class="ttdoc">Line iterator for files. </div><div class="ttdef"><b>Definition:</b> <a href="pio_8h_source.html#l00177">pio.h:177</a></div></div>
<div class="ttc" id="ckd__alloc_8h_html_ad313f92478859f9e4ea99d0f6e78c393"><div class="ttname"><a href="ckd__alloc_8h.html#ad313f92478859f9e4ea99d0f6e78c393">ckd_salloc</a></div><div class="ttdeci">#define ckd_salloc(ptr)</div><div class="ttdoc">Macro for ckd_salloc </div><div class="ttdef"><b>Definition:</b> <a href="ckd__alloc_8h_source.html#l00264">ckd_alloc.h:264</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_af0aa15288e06fc8271298e4fa7cdc91a"><div class="ttname"><a href="cmd__ln_8h.html#af0aa15288e06fc8271298e4fa7cdc91a">cmd_ln_str_r</a></div><div class="ttdeci">SPHINXBASE_EXPORT char const * cmd_ln_str_r(cmd_ln_t *cmdln, char const *name)</div><div class="ttdoc">Retrieve a string from a command-line object. </div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8c_source.html#l00949">cmd_ln.c:949</a></div></div>
<div class="ttc" id="ckd__alloc_8h_html_a31c6b405558620ac37599737b5722fbf"><div class="ttname"><a href="ckd__alloc_8h.html#a31c6b405558620ac37599737b5722fbf">ckd_free</a></div><div class="ttdeci">SPHINXBASE_EXPORT void ckd_free(void *ptr)</div><div class="ttdoc">Test and free a 1-D array. </div><div class="ttdef"><b>Definition:</b> <a href="ckd__alloc_8c_source.html#l00241">ckd_alloc.c:241</a></div></div>
<div class="ttc" id="err_8h_html_a5229a1d58f5f5e69963a8d038ff5bc3e"><div class="ttname"><a href="err_8h.html#a5229a1d58f5f5e69963a8d038ff5bc3e">E_FATAL_SYSTEM</a></div><div class="ttdeci">#define E_FATAL_SYSTEM</div><div class="ttdoc">Print error text; Call perror(""); exit(errno);. </div><div class="ttdef"><b>Definition:</b> <a href="err_8h_source.html#l00132">err.h:132</a></div></div>
<div class="ttc" id="logmath_8h_html_a5835860c5f6a703c80c0214f816f0b11"><div class="ttname"><a href="logmath_8h.html#a5835860c5f6a703c80c0214f816f0b11">logmath_init</a></div><div class="ttdeci">SPHINXBASE_EXPORT logmath_t * logmath_init(float64 base, int shift, int use_table)</div><div class="ttdoc">Initialize a log math computation table. </div><div class="ttdef"><b>Definition:</b> <a href="logmath_8c_source.html#l00062">logmath.c:62</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_ad9b1952e4f1def9ee6a88791375b3901"><div class="ttname"><a href="cmd__ln_8h.html#ad9b1952e4f1def9ee6a88791375b3901">ARG_FLOAT32</a></div><div class="ttdeci">#define ARG_FLOAT32</div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8h_source.html#l00148">cmd_ln.h:148</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_aceb617c8572cf5ad6257b35e6d8919e7"><div class="ttname"><a href="cmd__ln_8h.html#aceb617c8572cf5ad6257b35e6d8919e7">ARG_FLOAT64</a></div><div class="ttdeci">#define ARG_FLOAT64</div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8h_source.html#l00152">cmd_ln.h:152</a></div></div>
<div class="ttc" id="ngram__model_8h_html"><div class="ttname"><a href="ngram__model_8h.html">ngram_model.h</a></div><div class="ttdoc">N-Gram language models. </div></div>
<div class="ttc" id="ngram__model_8h_html_aa4b8d7c1f3d873b8458c0cfee13af4da"><div class="ttname"><a href="ngram__model_8h.html#aa4b8d7c1f3d873b8458c0cfee13af4da">ngram_model_apply_weights</a></div><div class="ttdeci">SPHINXBASE_EXPORT int ngram_model_apply_weights(ngram_model_t *model, float32 lw, float32 wip, float32 uw)</div><div class="ttdoc">Apply a language weight, insertion penalty, and unigram weight to a language model. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00477">ngram_model.c:477</a></div></div>
<div class="ttc" id="logmath_8h_html_a6114206ec0321d7015c42fc7b81cb83e"><div class="ttname"><a href="logmath_8h.html#a6114206ec0321d7015c42fc7b81cb83e">logmath_get_base</a></div><div class="ttdeci">SPHINXBASE_EXPORT float64 logmath_get_base(logmath_t *lmath)</div><div class="ttdoc">Get the log base. </div><div class="ttdef"><b>Definition:</b> <a href="logmath_8c_source.html#l00368">logmath.c:368</a></div></div>
<div class="ttc" id="pio_8h_html_aff8df0b6928746d61b3520555263f71e"><div class="ttname"><a href="pio_8h.html#aff8df0b6928746d61b3520555263f71e">lineiter_next</a></div><div class="ttdeci">SPHINXBASE_EXPORT lineiter_t * lineiter_next(lineiter_t *li)</div><div class="ttdoc">Move to the next line in the file. </div><div class="ttdef"><b>Definition:</b> <a href="pio_8c_source.html#l00338">pio.c:338</a></div></div>
<div class="ttc" id="pio_8h_html_a22d0125ab198f02f8bbe543417d99566"><div class="ttname"><a href="pio_8h.html#a22d0125ab198f02f8bbe543417d99566">lineiter_start</a></div><div class="ttdeci">SPHINXBASE_EXPORT lineiter_t * lineiter_start(FILE *fh)</div><div class="ttdoc">Start reading lines from a file. </div><div class="ttdef"><b>Definition:</b> <a href="pio_8c_source.html#l00255">pio.c:255</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a462d374099a4fe8b3c3195b5e2013545"><div class="ttname"><a href="ngram__model_8h.html#a462d374099a4fe8b3c3195b5e2013545">ngram_model_get_size</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 ngram_model_get_size(ngram_model_t *model)</div><div class="ttdoc">Get the order of the N-gram model (i.e. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00663">ngram_model.c:663</a></div></div>
<div class="ttc" id="err_8h_html"><div class="ttname"><a href="err_8h.html">err.h</a></div><div class="ttdoc">Implementation of logging routines. </div></div>
<div class="ttc" id="cmd__ln_8h_html_ac7d08ff59bb6905c3375162e75913e88"><div class="ttname"><a href="cmd__ln_8h.html#ac7d08ff59bb6905c3375162e75913e88">ARG_BOOLEAN</a></div><div class="ttdeci">#define ARG_BOOLEAN</div><div class="ttdoc">Boolean (true/false) argument (optional). </div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8h_source.html#l00118">cmd_ln.h:118</a></div></div>
<div class="ttc" id="structarg__t_html"><div class="ttname"><a href="structarg__t.html">arg_t</a></div><div class="ttdoc">Argument definition structure. </div></div>
<div class="ttc" id="ngram__model_8h_html_a9b2a86c23543158754373c5456fe890d"><div class="ttname"><a href="ngram__model_8h.html#a9b2a86c23543158754373c5456fe890d">ngram_model_read_classdef</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 ngram_model_read_classdef(ngram_model_t *model, const char *file_name)</div><div class="ttdoc">Read a class definition file and add classes to a language model. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l01199">ngram_model.c:1199</a></div></div>
<div class="ttc" id="strfuncs_8h_html_a5b520fdebcca599db86faaf75a82173f"><div class="ttname"><a href="strfuncs_8h.html#a5b520fdebcca599db86faaf75a82173f">str2words</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 str2words(char *line, char **wptr, int32 n_wptr)</div><div class="ttdoc">Convert a line to an array of "words", based on whitespace separators. </div><div class="ttdef"><b>Definition:</b> <a href="strfuncs_8c_source.html#l00115">strfuncs.c:115</a></div></div>
<div class="ttc" id="structcmd__ln__t_html"><div class="ttname"><a href="structcmd__ln__t.html">cmd_ln_t</a></div><div class="ttdoc">Opaque structure used to hold the results of command-line parsing. </div></div>
<div class="ttc" id="err_8h_html_a1a4495946ab2449d61108fe829a94613"><div class="ttname"><a href="err_8h.html#a1a4495946ab2449d61108fe829a94613">E_FATAL</a></div><div class="ttdeci">#define E_FATAL</div><div class="ttdoc">Exit with non-zero status after error message. </div><div class="ttdef"><b>Definition:</b> <a href="err_8h_source.html#l00127">err.h:127</a></div></div>
<div class="ttc" id="cmd__ln_8h_html_a159e691c95089689cf9a8f85a67830a6"><div class="ttname"><a href="cmd__ln_8h.html#a159e691c95089689cf9a8f85a67830a6">cmd_ln_boolean_r</a></div><div class="ttdeci">#define cmd_ln_boolean_r(c, n)</div><div class="ttdoc">Retrieve a boolean value from a command-line object. </div><div class="ttdef"><b>Definition:</b> <a href="cmd__ln_8h_source.html#l00334">cmd_ln.h:334</a></div></div>
<div class="ttc" id="structsphinx__wave2feat__s_html_a484d308befbde315664da8520ebc410d"><div class="ttname"><a href="structsphinx__wave2feat__s.html#a484d308befbde315664da8520ebc410d">sphinx_wave2feat_s::config</a></div><div class="ttdeci">cmd_ln_t * config</div><div class="ttdoc">Configuration parameters. </div><div class="ttdef"><b>Definition:</b> <a href="sphinx__fe_8c_source.html#l00078">sphinx_fe.c:78</a></div></div>
<div class="ttc" id="structngram__model__s_html"><div class="ttname"><a href="structngram__model__s.html">ngram_model_s</a></div><div class="ttdoc">Common implementation of ngram_model_t. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model__internal_8h_source.html#l00055">ngram_model_internal.h:55</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a96e36290a005c03464ea6c637ccde2f5"><div class="ttname"><a href="ngram__model_8h.html#a96e36290a005c03464ea6c637ccde2f5">ngram_word</a></div><div class="ttdeci">SPHINXBASE_EXPORT const char * ngram_word(ngram_model_t *model, int32 wid)</div><div class="ttdoc">Look up word string for numerical word ID. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00782">ngram_model.c:782</a></div></div>
<div class="ttc" id="logmath_8h_html"><div class="ttname"><a href="logmath_8h.html">logmath.h</a></div><div class="ttdoc">Fast integer logarithmic addition operations. </div></div>
<div class="ttc" id="logmath_8h_html_ae8b0a168e29e448c0d6de66dc46e099e"><div class="ttname"><a href="logmath_8h.html#ae8b0a168e29e448c0d6de66dc46e099e">logmath_exp</a></div><div class="ttdeci">SPHINXBASE_EXPORT float64 logmath_exp(logmath_t *lmath, int logb_p)</div><div class="ttdoc">Convert integer log in base B to linear floating point. </div><div class="ttdef"><b>Definition:</b> <a href="logmath_8c_source.html#l00456">logmath.c:456</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a406c0d64c15a9d1749d07c8ab6e0ae74a441701bf8ae0a2b79716feb31b5f257a"><div class="ttname"><a href="ngram__model_8h.html#a406c0d64c15a9d1749d07c8ab6e0ae74a441701bf8ae0a2b79716feb31b5f257a">NGRAM_AUTO</a></div><div class="ttdoc">Determine file type automatically. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8h_source.html#l00078">ngram_model.h:78</a></div></div>
<div class="ttc" id="ngram__model_8h_html_a6ac5799e78ea4ad82a11e2439016471e"><div class="ttname"><a href="ngram__model_8h.html#a6ac5799e78ea4ad82a11e2439016471e">ngram_ng_score</a></div><div class="ttdeci">SPHINXBASE_EXPORT int32 ngram_ng_score(ngram_model_t *model, int32 wid, int32 *history, int32 n_hist, int32 *n_used)</div><div class="ttdoc">Quick general N-Gram score lookup. </div><div class="ttdef"><b>Definition:</b> <a href="ngram__model_8c_source.html#l00494">ngram_model.c:494</a></div></div>
<div class="ttc" id="pio_8h_html"><div class="ttname"><a href="pio_8h.html">pio.h</a></div><div class="ttdoc">file IO related operations. </div></div>
</div><!-- fragment --></div><!-- contents -->
</div><!-- doc-content -->
<!-- start footer part -->
<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
<ul>
<li class="navelem"><a class="el" href="dir_68267d1309a1af8e8297ef4c3efbcdba.html">src</a></li><li class="navelem"><a class="el" href="dir_f6ff775926efe49187aa11a40d383708.html">sphinx_lmtools</a></li><li class="navelem"><a class="el" href="sphinx__lm__eval_8c.html">sphinx_lm_eval.c</a></li>
<li class="footer">Generated on Sat Oct 18 2014 15:21:17 for SphinxBase by
<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/></a> 1.8.8 </li>
</ul>
</div>
</body>
</html>
distrib
>
Mageia
>
5
>
x86_64
>
media
>
core-release
>
by-pkgid
>
b56c9eed3fa0f00740b6f09a2460a97f
>
files
>
410
