diff --git a/DeepSpeech.py b/DeepSpeech.py index ea65434566..3b8e98609d 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -18,12 +18,10 @@ from ds_ctcdecoder import ctc_beam_search_decoder, Scorer from six.moves import zip, range from tensorflow.python.tools import freeze_graph -from util.audio import audiofile_to_input_vector from util.config import Config, initialize_globals -from util.feeding import DataSet, ModelFeeder +from util.feeding import create_dataset, samples_to_mfccs, audiofile_to_features from util.flags import create_flags, FLAGS from util.logging import log_info, log_error, log_debug, log_warn -from util.preprocess import preprocess # Graph Creation @@ -42,25 +40,82 @@ def variable_on_cpu(name, shape, initializer): return var -def BiRNN(batch_x, seq_length, dropout, reuse=False, batch_size=None, n_steps=-1, previous_state=None, tflite=False): - r''' - That done, we will define the learned variables, the weights and biases, - within the method ``BiRNN()`` which also constructs the neural network. - The variables named ``hn``, where ``n`` is an integer, hold the learned weight variables. - The variables named ``bn``, where ``n`` is an integer, hold the learned bias variables. - In particular, the first variable ``h1`` holds the learned weight matrix that - converts an input vector of dimension ``n_input + 2*n_input*n_context`` - to a vector of dimension ``n_hidden_1``. - Similarly, the second variable ``h2`` holds the weight matrix converting - an input vector of dimension ``n_hidden_1`` to one of dimension ``n_hidden_2``. - The variables ``h3``, ``h5``, and ``h6`` are similar. - Likewise, the biases, ``b1``, ``b2``..., hold the biases for the various layers. - ''' +def create_overlapping_windows(batch_x): + batch_size = tf.shape(batch_x)[0] + window_width = 2 * Config.n_context + 1 + num_channels = Config.n_input + + # Create a constant convolution filter using an identity matrix, so that the + # convolution returns patches of the input tensor as is, and we can create + # overlapping windows over the MFCCs. + eye_filter = tf.constant(np.eye(window_width * num_channels) + .reshape(window_width, num_channels, window_width * num_channels), tf.float32) + + # Create overlapping windows + batch_x = tf.nn.conv1d(batch_x, eye_filter, stride=1, padding='SAME') + + # Remove dummy depth dimension and reshape into [batch_size, n_windows, window_width, n_input] + batch_x = tf.reshape(batch_x, [batch_size, -1, window_width, num_channels]) + + return batch_x + + +def dense(name, x, units, dropout_rate=None, relu=True): + with tf.variable_scope(name): + bias = variable_on_cpu('bias', [units], tf.zeros_initializer()) + weights = variable_on_cpu('weights', [x.shape[-1], units], tf.contrib.layers.xavier_initializer()) + + output = tf.nn.bias_add(tf.matmul(x, weights), bias) + + if relu: + output = tf.minimum(tf.nn.relu(output), FLAGS.relu_clip) + + if dropout_rate is not None: + output = tf.nn.dropout(output, rate=dropout_rate) + + return output + + +def rnn_impl_lstmblockfusedcell(x, seq_length, previous_state, reuse): + # Forward direction cell: + fw_cell = tf.contrib.rnn.LSTMBlockFusedCell(Config.n_cell_dim, reuse=reuse) + + output, output_state = fw_cell(inputs=x, + dtype=tf.float32, + sequence_length=seq_length, + initial_state=previous_state) + + return output, output_state + + +def rnn_impl_static_rnn(x, seq_length, previous_state, reuse): + # Forward direction cell: + fw_cell = tf.nn.rnn_cell.LSTMCell(Config.n_cell_dim, reuse=reuse) + + # Split rank N tensor into list of rank N-1 tensors + x = [x[l] for l in range(x.shape[0])] + + # We parametrize the RNN implementation as the training and inference graph + # need to do different things here. + output, output_state = tf.nn.static_rnn(cell=fw_cell, + inputs=x, + initial_state=previous_state, + dtype=tf.float32, + sequence_length=seq_length) + output = tf.concat(output, 0) + + return output, output_state + + +def create_model(batch_x, seq_length, dropout, reuse=False, previous_state=None, overlap=True, rnn_impl=rnn_impl_lstmblockfusedcell): layers = {} # Input shape: [batch_size, n_steps, n_input + 2*n_input*n_context] - if not batch_size: - batch_size = tf.shape(batch_x)[0] + batch_size = tf.shape(batch_x)[0] + + # Create overlapping feature windows if needed + if overlap: + batch_x = create_overlapping_windows(batch_x) # Reshaping `batch_x` to a tensor with shape `[n_steps*batch_size, n_input + 2*n_input*n_context]`. # This is done to prepare the batch for input into the first layer which expects a tensor of rank `2`. @@ -73,58 +128,17 @@ def BiRNN(batch_x, seq_length, dropout, reuse=False, batch_size=None, n_steps=-1 # The next three blocks will pass `batch_x` through three hidden layers with # clipped RELU activation and dropout. - - # 1st layer - b1 = variable_on_cpu('b1', [Config.n_hidden_1], tf.zeros_initializer()) - h1 = variable_on_cpu('h1', [Config.n_input + 2*Config.n_input*Config.n_context, Config.n_hidden_1], tf.contrib.layers.xavier_initializer()) - layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), FLAGS.relu_clip) - layer_1 = tf.nn.dropout(layer_1, rate=dropout[0]) - layers['layer_1'] = layer_1 - - # 2nd layer - b2 = variable_on_cpu('b2', [Config.n_hidden_2], tf.zeros_initializer()) - h2 = variable_on_cpu('h2', [Config.n_hidden_1, Config.n_hidden_2], tf.contrib.layers.xavier_initializer()) - layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), FLAGS.relu_clip) - layer_2 = tf.nn.dropout(layer_2, rate=dropout[1]) - layers['layer_2'] = layer_2 - - # 3rd layer - b3 = variable_on_cpu('b3', [Config.n_hidden_3], tf.zeros_initializer()) - h3 = variable_on_cpu('h3', [Config.n_hidden_2, Config.n_hidden_3], tf.contrib.layers.xavier_initializer()) - layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), FLAGS.relu_clip) - layer_3 = tf.nn.dropout(layer_3, rate=dropout[2]) - layers['layer_3'] = layer_3 - - # Now we create the forward and backward LSTM units. - # Both of which have inputs of length `n_cell_dim` and bias `1.0` for the forget gate of the LSTM. - - # Forward direction cell: - if not tflite: - fw_cell = tf.contrib.rnn.LSTMBlockFusedCell(Config.n_cell_dim, reuse=reuse) - layers['fw_cell'] = fw_cell - else: - fw_cell = tf.nn.rnn_cell.LSTMCell(Config.n_cell_dim, reuse=reuse) + layers['layer_1'] = layer_1 = dense('layer_1', batch_x, Config.n_hidden_1, dropout_rate=dropout[0]) + layers['layer_2'] = layer_2 = dense('layer_2', layer_1, Config.n_hidden_2, dropout_rate=dropout[1]) + layers['layer_3'] = layer_3 = dense('layer_3', layer_2, Config.n_hidden_3, dropout_rate=dropout[2]) # `layer_3` is now reshaped into `[n_steps, batch_size, 2*n_cell_dim]`, # as the LSTM RNN expects its input to be of shape `[max_time, batch_size, input_size]`. - layer_3 = tf.reshape(layer_3, [n_steps, batch_size, Config.n_hidden_3]) - if tflite: - # Generated StridedSlice, not supported by NNAPI - #n_layer_3 = [] - #for l in range(layer_3.shape[0]): - # n_layer_3.append(layer_3[l]) - #layer_3 = n_layer_3 - - # Unstack/Unpack is not supported by NNAPI - layer_3 = tf.unstack(layer_3, n_steps) + layer_3 = tf.reshape(layer_3, [-1, batch_size, Config.n_hidden_3]) - # We parametrize the RNN implementation as the training and inference graph - # need to do different things here. - if not tflite: - output, output_state = fw_cell(inputs=layer_3, dtype=tf.float32, sequence_length=seq_length, initial_state=previous_state) - else: - output, output_state = tf.nn.static_rnn(fw_cell, layer_3, previous_state, tf.float32) - output = tf.concat(output, 0) + # Run through parametrized RNN implementation, as we use different RNNs + # for training and inference + output, output_state = rnn_impl(layer_3, seq_length, previous_state, reuse) # Reshape output from a tensor of shape [n_steps, batch_size, n_cell_dim] # to a tensor of shape [n_steps*batch_size, n_cell_dim] @@ -132,24 +146,16 @@ def BiRNN(batch_x, seq_length, dropout, reuse=False, batch_size=None, n_steps=-1 layers['rnn_output'] = output layers['rnn_output_state'] = output_state - # Now we feed `output` to the fifth hidden layer with clipped RELU activation and dropout - b5 = variable_on_cpu('b5', [Config.n_hidden_5], tf.zeros_initializer()) - h5 = variable_on_cpu('h5', [Config.n_cell_dim, Config.n_hidden_5], tf.contrib.layers.xavier_initializer()) - layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(output, h5), b5)), FLAGS.relu_clip) - layer_5 = tf.nn.dropout(layer_5, rate=dropout[5]) - layers['layer_5'] = layer_5 + # Now we feed `output` to the fifth hidden layer with clipped RELU activation + layers['layer_5'] = layer_5 = dense('layer_5', output, Config.n_hidden_5, dropout_rate=dropout[5]) - # Now we apply the weight matrix `h6` and bias `b6` to the output of `layer_5` - # creating `n_classes` dimensional vectors, the logits. - b6 = variable_on_cpu('b6', [Config.n_hidden_6], tf.zeros_initializer()) - h6 = variable_on_cpu('h6', [Config.n_hidden_5, Config.n_hidden_6], tf.contrib.layers.xavier_initializer()) - layer_6 = tf.add(tf.matmul(layer_5, h6), b6) - layers['layer_6'] = layer_6 + # Now we apply a final linear layer creating `n_classes` dimensional vectors, the logits. + layers['layer_6'] = layer_6 = dense('layer_6', layer_5, Config.n_hidden_6, relu=False) # Finally we reshape layer_6 from a tensor of shape [n_steps*batch_size, n_hidden_6] # to the slightly more useful shape [n_steps, batch_size, n_hidden_6]. # Note, that this differs from the input in that it is time-major. - layer_6 = tf.reshape(layer_6, [n_steps, batch_size, Config.n_hidden_6], name="raw_logits") + layer_6 = tf.reshape(layer_6, [-1, batch_size, Config.n_hidden_6], name='raw_logits') layers['raw_logits'] = layer_6 # Output shape: [n_steps, batch_size, n_hidden_6] @@ -166,17 +172,17 @@ def BiRNN(batch_x, seq_length, dropout, reuse=False, batch_size=None, n_steps=-1 # Conveniently, this loss function is implemented in TensorFlow. # Thus, we can simply make use of this implementation to define our loss. -def calculate_mean_edit_distance_and_loss(model_feeder, tower, dropout, reuse): +def calculate_mean_edit_distance_and_loss(iterator, tower, dropout, reuse): r''' This routine beam search decodes a mini-batch and calculates the loss and mean edit distance. Next to total and average loss it returns the mean edit distance, the decoded result and the batch's original Y. ''' # Obtain the next batch of data - batch_x, batch_seq_len, batch_y = model_feeder.next_batch(tower) + (batch_x, batch_seq_len), batch_y = iterator.get_next() - # Calculate the logits of the batch using BiRNN - logits, _ = BiRNN(batch_x, batch_seq_len, dropout, reuse) + # Calculate the logits of the batch + logits, _ = create_model(batch_x, batch_seq_len, dropout, reuse=reuse) # Compute the CTC loss using TensorFlow's `ctc_loss` total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len) @@ -221,7 +227,7 @@ def create_optimizer(): # on which all operations within the tower execute. # For example, all operations of 'tower 0' could execute on the first GPU `tf.device('/gpu:0')`. -def get_tower_results(model_feeder, optimizer, dropout_rates): +def get_tower_results(iterator, optimizer, dropout_rates): r''' With this preliminary step out of the way, we can for each GPU introduce a tower for which's batch we calculate and return the optimization gradients @@ -243,7 +249,7 @@ def get_tower_results(model_feeder, optimizer, dropout_rates): with tf.name_scope('tower_%d' % i) as scope: # Calculate the avg_loss and mean_edit_distance and retrieve the decoded # batch along with the original batch's labels (Y) of this tower - avg_loss = calculate_mean_edit_distance_and_loss(model_feeder, i, dropout_rates, reuse=i>0) + avg_loss = calculate_mean_edit_distance_and_loss(iterator, i, dropout_rates, reuse=i>0) # Allow for variables to be re-used by the next tower tf.get_variable_scope().reuse_variables() @@ -313,7 +319,7 @@ def log_variable(variable, gradient=None): It logs scalar values for the mean, standard deviation, minimum and maximum. Furthermore it logs a histogram of its state and (if given) of an optimization gradient. ''' - name = variable.name + name = variable.name.replace(':', '_') mean = tf.reduce_mean(variable) tf.summary.scalar(name='%s/mean' % name, tensor=mean) tf.summary.scalar(name='%s/sttdev' % name, tensor=tf.sqrt(tf.reduce_mean(tf.square(variable - mean)))) @@ -337,19 +343,6 @@ def log_grads_and_vars(grads_and_vars): log_variable(variable, gradient=gradient) -# Helpers -# ======= - - -class SampleIndex: - def __init__(self, index=0): - self.index = index - - def inc(self, old_index): - self.index += 1 - return self.index - - def try_loading(session, saver, checkpoint_filename, caption): try: checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir, checkpoint_filename) @@ -371,48 +364,23 @@ def try_loading(session, saver, checkpoint_filename, caption): def train(): - r''' - Trains the network on a given server of a cluster. - If no server provided, it performs single process training. - ''' + # Create training and validation datasets + train_set, train_batches = create_dataset(FLAGS.train_files.split(','), + batch_size=FLAGS.train_batch_size, + cache_path=FLAGS.train_cached_features_path) + + iterator = tf.data.Iterator.from_structure(train_set.output_types, + train_set.output_shapes, + output_classes=train_set.output_classes) - # Reading training set - train_index = SampleIndex() - - train_data = preprocess(FLAGS.train_files.split(','), - FLAGS.train_batch_size, - Config.n_input, - Config.n_context, - Config.alphabet, - hdf5_cache_path=FLAGS.train_cached_features_path) - - train_set = DataSet(train_data, - FLAGS.train_batch_size, - limit=FLAGS.limit_train, - next_index=train_index.inc) - - # Reading validation set - dev_index = SampleIndex() - - dev_data = preprocess(FLAGS.dev_files.split(','), - FLAGS.dev_batch_size, - Config.n_input, - Config.n_context, - Config.alphabet, - hdf5_cache_path=FLAGS.dev_cached_features_path) - - dev_set = DataSet(dev_data, - FLAGS.dev_batch_size, - limit=FLAGS.limit_dev, - next_index=dev_index.inc) - - # Combining all sets to a multi set model feeder - model_feeder = ModelFeeder(train_set, - dev_set, - Config.n_input, - Config.n_context, - Config.alphabet, - tower_feeder_count=len(Config.available_devices)) + # Make initialization ops for switching between the two sets + train_init_op = iterator.make_initializer(train_set) + + if FLAGS.dev_files: + dev_set, dev_batches = create_dataset(FLAGS.dev_files.split(','), + batch_size=FLAGS.dev_batch_size, + cache_path=FLAGS.dev_cached_features_path) + dev_init_op = iterator.make_initializer(dev_set) # Dropout dropout_rates = [tf.placeholder(tf.float32, name='dropout_{}'.format(i)) for i in range(6)] @@ -425,17 +393,12 @@ def train(): dropout_rates[5]: FLAGS.dropout_rate6, } no_dropout_feed_dict = { - dropout_rates[0]: 0., - dropout_rates[1]: 0., - dropout_rates[2]: 0., - dropout_rates[3]: 0., - dropout_rates[4]: 0., - dropout_rates[5]: 0., + rate: 0. for rate in dropout_rates } # Building the graph optimizer = create_optimizer() - gradients, loss = get_tower_results(model_feeder, optimizer, dropout_rates) + gradients, loss = get_tower_results(iterator, optimizer, dropout_rates) # Average tower gradients across GPUs avg_tower_gradients = average_gradients(gradients) log_grads_and_vars(avg_tower_gradients) @@ -463,6 +426,7 @@ def train(): with tf.Session(config=Config.session_config) as session: log_debug('Session opened.') + tf.get_default_graph().finalize() # Loading or initializing @@ -481,51 +445,51 @@ def train(): sys.exit(1) # Retrieving global_step from restored model and setting training parameters accordingly - model_feeder.set_data_set(no_dropout_feed_dict, train_set) - step = session.run(global_step, feed_dict=no_dropout_feed_dict) + step = session.run(global_step) num_gpus = len(Config.available_devices) - steps_per_epoch = max(1, train_set.total_batches // num_gpus) - steps_trained = step % steps_per_epoch + steps_per_epoch = max(1, train_batches // num_gpus) current_epoch = step // steps_per_epoch target_epoch = current_epoch + abs(FLAGS.epoch) if FLAGS.epoch < 0 else FLAGS.epoch - train_index.index = steps_trained * num_gpus log_debug('step: %d' % step) log_debug('epoch: %d' % current_epoch) log_debug('target epoch: %d' % target_epoch) log_debug('steps per epoch: %d' % steps_per_epoch) log_debug('batches per step (GPUs): %d' % num_gpus) - log_debug('number of batches in train set: %d' % train_set.total_batches) - log_debug('number of batches already trained in epoch: %d' % train_index.index) + log_debug('number of batches in train set: %d' % train_batches) - def run_set(set_name): - data_set = getattr(model_feeder, set_name) + def run_set(set_name, init_op, num_batches): is_train = set_name == 'train' train_op = apply_gradient_op if is_train else [] feed_dict = dropout_feed_dict if is_train else no_dropout_feed_dict - model_feeder.set_data_set(feed_dict, data_set) total_loss = 0.0 step_summary_writer = step_summary_writers.get(set_name) - num_steps = max(1, data_set.total_batches // num_gpus) + num_steps = max(1, num_batches // num_gpus) checkpoint_time = time.time() + if FLAGS.show_progressbar: pbar = progressbar.ProgressBar(max_value=num_steps, redirect_stdout=True).start() + else: + pbar = lambda i: i + + # Initialize iterator to the appropriate dataset + session.run(init_op) + # Batch loop - for step_index in range(steps_trained, num_steps): + for step_index in pbar(range(num_steps)): if coord.should_stop(): break + _, current_step, batch_loss, step_summary = \ session.run([train_op, global_step, loss, step_summaries_op], feed_dict=feed_dict) total_loss += batch_loss step_summary_writer.add_summary(step_summary, current_step) - if FLAGS.show_progressbar: - pbar.update(step_index + 1, force=True) + if is_train and FLAGS.checkpoint_secs > 0 and time.time() - checkpoint_time > FLAGS.checkpoint_secs: checkpoint_saver.save(session, checkpoint_path, global_step=current_step) checkpoint_time = time.time() - if FLAGS.show_progressbar: - pbar.finish() + return total_loss / num_steps if target_epoch > current_epoch: @@ -534,68 +498,66 @@ def run_set(set_name): dev_losses = [] coord = tf.train.Coordinator() with coord.stop_on_exception(): - log_debug('Starting queue runners...') - model_feeder.start_queue_threads(session, coord=coord) - log_debug('Queue runners started.') - # Epoch loop for current_epoch in range(current_epoch, target_epoch): - # Training if coord.should_stop(): break + + # Training log_info('Training epoch %d ...' % current_epoch) - train_loss = run_set('train') + train_loss = run_set('train', train_init_op, train_batches) log_info('Finished training epoch %d - loss: %f' % (current_epoch, train_loss)) checkpoint_saver.save(session, checkpoint_path, global_step=global_step) - steps_trained = 0 - # Validation - log_info('Validating epoch %d ...' % current_epoch) - dev_loss = run_set('dev') - dev_losses.append(dev_loss) - log_info('Finished validating epoch %d - loss: %f' % (current_epoch, dev_loss)) - if dev_loss < best_dev_loss: - best_dev_loss = dev_loss - save_path = best_dev_saver.save(session, best_dev_path, latest_filename=best_dev_filename) - log_info("Saved new best validating model with loss %f to: %s" % (best_dev_loss, save_path)) - # Early stopping - if FLAGS.early_stop and len(dev_losses) >= FLAGS.es_steps: - mean_loss = np.mean(dev_losses[-FLAGS.es_steps:-1]) - std_loss = np.std(dev_losses[-FLAGS.es_steps:-1]) - dev_losses = dev_losses[-FLAGS.es_steps:] - log_debug('Checking for early stopping (last %d steps) validation loss: ' - '%f, with standard deviation: %f and mean: %f' % - (FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss)) - if dev_losses[-1] > np.max(dev_losses[:-1]) or \ - (abs(dev_losses[-1] - mean_loss) < FLAGS.es_mean_th and std_loss < FLAGS.es_std_th): - log_info('Early stop triggered as (for last %d steps) validation loss:' - ' %f with standard deviation: %f and mean: %f' % - (FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss)) - break - log_debug('Closing queues...') + + if FLAGS.dev_files: + # Validation + log_info('Validating epoch %d ...' % current_epoch) + dev_loss = run_set('dev', dev_init_op, dev_batches) + dev_losses.append(dev_loss) + log_info('Finished validating epoch %d - loss: %f' % (current_epoch, dev_loss)) + + if dev_loss < best_dev_loss: + best_dev_loss = dev_loss + save_path = best_dev_saver.save(session, best_dev_path, latest_filename=best_dev_filename) + log_info("Saved new best validating model with loss %f to: %s" % (best_dev_loss, save_path)) + + # Early stopping + if FLAGS.early_stop and len(dev_losses) >= FLAGS.es_steps: + mean_loss = np.mean(dev_losses[-FLAGS.es_steps:-1]) + std_loss = np.std(dev_losses[-FLAGS.es_steps:-1]) + dev_losses = dev_losses[-FLAGS.es_steps:] + log_debug('Checking for early stopping (last %d steps) validation loss: ' + '%f, with standard deviation: %f and mean: %f' % + (FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss)) + if dev_losses[-1] > np.max(dev_losses[:-1]) or \ + (abs(dev_losses[-1] - mean_loss) < FLAGS.es_mean_th and std_loss < FLAGS.es_std_th): + log_info('Early stop triggered as (for last %d steps) validation loss:' + ' %f with standard deviation: %f and mean: %f' % + (FLAGS.es_steps, dev_losses[-1], std_loss, mean_loss)) + break coord.request_stop() - model_feeder.close_queues(session) - log_debug('Queues closed.') else: log_info('Target epoch already reached - skipped training.') log_debug('Session closed.') def test(): - # Reading test set - test_data = preprocess(FLAGS.test_files.split(','), - FLAGS.test_batch_size, - Config.n_input, - Config.n_context, - Config.alphabet, - hdf5_cache_path=FLAGS.test_cached_features_path) - - graph = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=-1) - evaluate.evaluate(test_data, graph) + evaluate.evaluate(FLAGS.test_files.split(','), create_model) def create_inference_graph(batch_size=1, n_steps=16, tflite=False): batch_size = batch_size if batch_size > 0 else None + + # Create feature computation graph + input_samples = tf.placeholder(tf.float32, [512], 'input_samples') + samples = tf.expand_dims(input_samples, -1) + mfccs, _ = samples_to_mfccs(samples, 16000) + mfccs = tf.identity(mfccs, name='mfccs') + # Input tensor will be of shape [batch_size, n_steps, 2*n_context+1, n_input] - input_tensor = tf.placeholder(tf.float32, [batch_size, n_steps if n_steps > 0 else None, 2*Config.n_context+1, Config.n_input], name='input_node') + # This shape is read by the native_client in DS_CreateModel to know the + # value of n_steps, n_context and n_input. Make sure you update the code + # there if this shape is changed. + input_tensor = tf.placeholder(tf.float32, [batch_size, n_steps if n_steps > 0 else None, 2 * Config.n_context + 1, Config.n_input], name='input_node') seq_length = tf.placeholder(tf.int32, [batch_size], name='input_lengths') if batch_size <= 0: @@ -611,15 +573,20 @@ def create_inference_graph(batch_size=1, n_steps=16, tflite=False): previous_state = tf.contrib.rnn.LSTMStateTuple(previous_state_c, previous_state_h) - no_dropout = [0.0] * 6 + # One rate per layer + no_dropout = [None] * 6 + + if tflite: + rnn_impl = rnn_impl_static_rnn + else: + rnn_impl = rnn_impl_lstmblockfusedcell - logits, layers = BiRNN(batch_x=input_tensor, - seq_length=seq_length if FLAGS.use_seq_length else None, - dropout=no_dropout, - batch_size=batch_size, - n_steps=n_steps, - previous_state=previous_state, - tflite=tflite) + logits, layers = create_model(batch_x=input_tensor, + seq_length=seq_length if FLAGS.use_seq_length else None, + dropout=no_dropout, + previous_state=previous_state, + overlap=False, + rnn_impl=rnn_impl) # TF Lite runtime will check that input dimensions are 1, 2 or 4 # by default we get 3, the middle one being batch_size which is forced to @@ -659,10 +626,12 @@ def create_inference_graph(batch_size=1, n_steps=16, tflite=False): { 'input': input_tensor, 'input_lengths': seq_length, + 'input_samples': input_samples, }, { 'outputs': logits, 'initialize_state': initialize_state, + 'mfccs': mfccs, }, layers ) @@ -671,98 +640,111 @@ def create_inference_graph(batch_size=1, n_steps=16, tflite=False): new_state_c = tf.identity(new_state_c, name='new_state_c') new_state_h = tf.identity(new_state_h, name='new_state_h') + inputs = { + 'input': input_tensor, + 'previous_state_c': previous_state_c, + 'previous_state_h': previous_state_h, + 'input_samples': input_samples, + } + + if FLAGS.use_seq_length: + inputs.update({'input_lengths': seq_length}) + return ( - { - 'input': input_tensor, - 'previous_state_c': previous_state_c, - 'previous_state_h': previous_state_h, - }, + inputs, { 'outputs': logits, 'new_state_c': new_state_c, 'new_state_h': new_state_h, + 'mfccs': mfccs, }, layers ) +def file_relative_read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + def export(): r''' Restores the trained variables into a simpler graph that will be exported for serving. ''' log_info('Exporting the model...') - with tf.device('/cpu:0'): - from tensorflow.python.framework.ops import Tensor, Operation + from tensorflow.python.framework.ops import Tensor, Operation - tf.reset_default_graph() - session = tf.Session(config=Config.session_config) + inputs, outputs, _ = create_inference_graph(batch_size=FLAGS.export_batch_size, n_steps=FLAGS.n_steps, tflite=FLAGS.export_tflite) + input_names = ",".join(tensor.op.name for tensor in inputs.values()) + output_names_tensors = [ tensor.op.name for tensor in outputs.values() if isinstance(tensor, Tensor)] + output_names_ops = [ tensor.name for tensor in outputs.values() if isinstance(tensor, Operation)] + output_names = ",".join(output_names_tensors + output_names_ops) + input_shapes = ":".join(",".join(map(str, tensor.shape)) for tensor in inputs.values()) - inputs, outputs, _ = create_inference_graph(batch_size=FLAGS.export_batch_size, n_steps=FLAGS.n_steps, tflite=FLAGS.export_tflite) - input_names = ",".join(tensor.op.name for tensor in inputs.values()) - output_names_tensors = [ tensor.op.name for tensor in outputs.values() if isinstance(tensor, Tensor) ] - output_names_ops = [ tensor.name for tensor in outputs.values() if isinstance(tensor, Operation) ] - output_names = ",".join(output_names_tensors + output_names_ops) - input_shapes = ":".join(",".join(map(str, tensor.shape)) for tensor in inputs.values()) + if not FLAGS.export_tflite: + mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')} + else: + # Create a saver using variables from the above newly created graph + def fixup(name): + if name.startswith('rnn/lstm_cell/'): + return name.replace('rnn/lstm_cell/', 'lstm_fused_cell/') + return name - if not FLAGS.export_tflite: - mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')} - else: - # Create a saver using variables from the above newly created graph - def fixup(name): - if name.startswith('rnn/lstm_cell/'): - return name.replace('rnn/lstm_cell/', 'lstm_fused_cell/') - return name + mapping = {fixup(v.op.name): v for v in tf.global_variables()} - mapping = {fixup(v.op.name): v for v in tf.global_variables()} + saver = tf.train.Saver(mapping) - saver = tf.train.Saver(mapping) + # Restore variables from training checkpoint + checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) + checkpoint_path = checkpoint.model_checkpoint_path - # Restore variables from training checkpoint - checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) - checkpoint_path = checkpoint.model_checkpoint_path + output_filename = 'output_graph.pb' + if FLAGS.remove_export: + if os.path.isdir(FLAGS.export_dir): + log_info('Removing old export') + shutil.rmtree(FLAGS.export_dir) + try: + output_graph_path = os.path.join(FLAGS.export_dir, output_filename) + + if not os.path.isdir(FLAGS.export_dir): + os.makedirs(FLAGS.export_dir) + + def do_graph_freeze(output_file=None, output_node_names=None, variables_blacklist=None): + return freeze_graph.freeze_graph_with_def_protos( + input_graph_def=tf.get_default_graph().as_graph_def(), + input_saver_def=saver.as_saver_def(), + input_checkpoint=checkpoint_path, + output_node_names=output_node_names, + restore_op_name=None, + filename_tensor_name=None, + output_graph=output_file, + clear_devices=False, + variable_names_blacklist=variables_blacklist, + initializer_nodes='') - output_filename = 'output_graph.pb' - if FLAGS.remove_export: - if os.path.isdir(FLAGS.export_dir): - log_info('Removing old export') - shutil.rmtree(FLAGS.export_dir) - try: - output_graph_path = os.path.join(FLAGS.export_dir, output_filename) - - if not os.path.isdir(FLAGS.export_dir): - os.makedirs(FLAGS.export_dir) - - def do_graph_freeze(output_file=None, output_node_names=None, variables_blacklist=None): - return freeze_graph.freeze_graph_with_def_protos( - input_graph_def=session.graph_def, - input_saver_def=saver.as_saver_def(), - input_checkpoint=checkpoint_path, - output_node_names=output_node_names, - restore_op_name=None, - filename_tensor_name=None, - output_graph=output_file, - clear_devices=False, - variable_names_blacklist=variables_blacklist, - initializer_nodes='') - - if not FLAGS.export_tflite: - do_graph_freeze(output_file=output_graph_path, output_node_names=output_names, variables_blacklist='previous_state_c,previous_state_h') - else: - frozen_graph = do_graph_freeze(output_node_names=output_names, variables_blacklist='') - output_tflite_path = os.path.join(FLAGS.export_dir, output_filename.replace('.pb', '.tflite')) + if not FLAGS.export_tflite: + frozen_graph = do_graph_freeze(output_node_names=output_names, variables_blacklist='previous_state_c,previous_state_h') + frozen_graph.version = int(file_relative_read('GRAPH_VERSION').strip()) + with open(output_graph_path, 'wb') as fout: + fout.write(frozen_graph.SerializeToString()) + else: + frozen_graph = do_graph_freeze(output_node_names=output_names, variables_blacklist='') + output_tflite_path = os.path.join(FLAGS.export_dir, output_filename.replace('.pb', '.tflite')) + + converter = tf.lite.TFLiteConverter(frozen_graph, input_tensors=inputs.values(), output_tensors=outputs.values()) + converter.post_training_quantize = True + # AudioSpectrogram and Mfcc ops are custom but have built-in kernels in TFLite + converter.allow_custom_ops = True + tflite_model = converter.convert() - converter = tf.lite.TFLiteConverter(frozen_graph, input_tensors=inputs.values(), output_tensors=outputs.values()) - converter.post_training_quantize = True - tflite_model = converter.convert() + with open(output_tflite_path, 'wb') as fout: + fout.write(tflite_model) - with open(output_tflite_path, 'wb') as fout: - fout.write(tflite_model) + log_info('Exported model for TF Lite engine as {}'.format(os.path.basename(output_tflite_path))) - log_info('Exported model for TF Lite engine as {}'.format(os.path.basename(output_tflite_path))) + log_info('Models exported at %s' % (FLAGS.export_dir)) + except RuntimeError as e: + log_error(str(e)) - log_info('Models exported at %s' % (FLAGS.export_dir)) - except RuntimeError as e: - log_error(str(e)) def do_single_file_inference(input_file_path): with tf.Session(config=Config.session_config) as session: @@ -784,22 +766,20 @@ def do_single_file_inference(input_file_path): saver.restore(session, checkpoint_path) session.run(outputs['initialize_state']) - features = audiofile_to_input_vector(input_file_path, Config.n_input, Config.n_context) - num_strides = len(features) - (Config.n_context * 2) + features, features_len = audiofile_to_features(input_file_path) + + # Add batch dimension + features = tf.expand_dims(features, 0) + features_len = tf.expand_dims(features_len, 0) - # Create a view into the array with overlapping strides of size - # numcontext (past) + 1 (present) + numcontext (future) - window_size = 2*Config.n_context+1 - features = np.lib.stride_tricks.as_strided( - features, - (num_strides, window_size, Config.n_input), - (features.strides[0], features.strides[0], features.strides[1]), - writeable=False) + # Evaluate + features = create_overlapping_windows(features).eval(session=session) + features_len = features_len.eval(session=session) - logits = session.run(outputs['outputs'], feed_dict = { - inputs['input']: [features], - inputs['input_lengths']: [num_strides], - }) + logits = outputs['outputs'].eval(feed_dict={ + inputs['input']: features, + inputs['input_lengths']: features_len, + }, session=session) logits = np.squeeze(logits) @@ -814,19 +794,21 @@ def do_single_file_inference(input_file_path): def main(_): initialize_globals() - if FLAGS.train: - with tf.Graph().as_default(): - tf.set_random_seed(FLAGS.random_seed) - train() + if FLAGS.train_files: + tf.reset_default_graph() + tf.set_random_seed(FLAGS.random_seed) + train() - if FLAGS.test: - with tf.Graph().as_default(): - test() + if FLAGS.test_files: + tf.reset_default_graph() + test() if FLAGS.export_dir: + tf.reset_default_graph() export() - if len(FLAGS.one_shot_infer): + if FLAGS.one_shot_infer: + tf.reset_default_graph() do_single_file_inference(FLAGS.one_shot_infer) if __name__ == '__main__' : diff --git a/GRAPH_VERSION b/GRAPH_VERSION new file mode 100644 index 0000000000..56a6051ca2 --- /dev/null +++ b/GRAPH_VERSION @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/README.md b/README.md index 228c46dac4..f657790801 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,11 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech - [Prerequisites](#prerequisites) - [Getting the code](#getting-the-code) - [Getting the pre-trained model](#getting-the-pre-trained-model) -- [CUDA dependency](#cuda-dependency) - [Using the model](#using-the-model) + - [CUDA dependency](#cuda-dependency) + - [Model compatibility](#model-compatibility) - [Using the Python package](#using-the-python-package) - - [Using the command line client](#using-the-command-line-client) + - [Using the command-line client](#using-the-command-line-client) - [Using the Node.JS package](#using-the-nodejs-package) - [Installing bindings from source](#installing-bindings-from-source) - [Third party bindings](#third-party-bindings) @@ -48,6 +49,7 @@ See the output of `deepspeech -h` for more information on the use of `deepspeech - [Checkpointing](#checkpointing) - [Exporting a model for inference](#exporting-a-model-for-inference) - [Exporting a model for TFLite](#exporting-a-model-for-tflite) + - [Making a mmap-able model for inference](#making-a-mmap-able-model-for-inference) - [Continuing training from a release model](#continuing-training-from-a-release-model) - [Contact/Getting Help](#contactgetting-help) @@ -88,6 +90,10 @@ There are three ways to use DeepSpeech inference: The GPU capable builds (Python, NodeJS, C++ etc) depend on the same CUDA runtime as upstream TensorFlow. Currently with TensorFlow r1.12 it depends on CUDA 9.0 and CuDNN v7.2. +### Model compatibility + +DeepSpeech models are versioned to keep you from trying to use an incompatible graph with a newer client after a breaking change was made to the code. If you get an error saying your model file version is too old for the client, you should either upgrade to a newer model release, re-export your model from the checkpoint using a newer version of the code, or downgrade your client if you need to use the old model and can't re-export it. + ### Using the Python package Pre-built binaries which can be used for performing inference with a trained model can be installed with `pip3`. You can then use the `deepspeech` binary to do speech-to-text on an audio file: @@ -323,7 +329,7 @@ Refer to the corresponding [README.md](native_client/README.md) for information ### Exporting a model for TFLite -If you want to experiment with the TF Lite engine, you need to export a model that is compatible with it, then use the `--export_tflite` flag. If you already have a trained model, you can re-export it for TFLite by running `DeepSpeech.py` again and specifying the same `checkpoint_dir` that you used for training, as well as passing `--notrain --notest --export_tflite --export_dir /model/export/destination`. +If you want to experiment with the TF Lite engine, you need to export a model that is compatible with it, then use the `--nouse_seq_length --export_tflite` flags. If you already have a trained model, you can re-export it for TFLite by running `DeepSpeech.py` again and specifying the same `checkpoint_dir` that you used for training, as well as passing `--nouse_seq_length --export_tflite --export_dir /model/export/destination`. ### Making a mmap-able model for inference diff --git a/bin/run-ldc93s1.sh b/bin/run-ldc93s1.sh index a48a85e533..4a6527e8dc 100755 --- a/bin/run-ldc93s1.sh +++ b/bin/run-ldc93s1.sh @@ -16,12 +16,10 @@ else checkpoint_dir=$(python -c 'from xdg import BaseDirectory as xdg; print(xdg.save_data_path("deepspeech/ldc93s1"))') fi -python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ +python -u DeepSpeech.py --noshow_progressbar \ --train_files data/ldc93s1/ldc93s1.csv \ - --dev_files data/ldc93s1/ldc93s1.csv \ --test_files data/ldc93s1/ldc93s1.csv \ --train_batch_size 1 \ - --dev_batch_size 1 \ --test_batch_size 1 \ --n_hidden 100 \ --epoch 200 \ diff --git a/bin/run-tc-ldc93s1_checkpoint.sh b/bin/run-tc-ldc93s1_checkpoint.sh index e8fdee8f33..de92a30459 100755 --- a/bin/run-tc-ldc93s1_checkpoint.sh +++ b/bin/run-tc-ldc93s1_checkpoint.sh @@ -16,7 +16,7 @@ python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ --train_files ${ldc93s1_csv} --train_batch_size 1 \ --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch -1 --random_seed 4567 --default_stddev 0.046875 \ + --n_hidden 100 --epoch -1 \ --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' \ --learning_rate 0.001 --dropout_rate 0.05 \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ diff --git a/bin/run-tc-ldc93s1_new.sh b/bin/run-tc-ldc93s1_new.sh index cee8ede403..f1e64d3e3c 100755 --- a/bin/run-tc-ldc93s1_new.sh +++ b/bin/run-tc-ldc93s1_new.sh @@ -14,12 +14,11 @@ fi; python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ --train_files ${ldc93s1_csv} --train_batch_size 1 \ - --train_cached_features_path "/tmp/ldc93s1.hdf5" \ + --train_cached_features_path '/tmp/ldc93s1_cache' \ --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch $epoch_count --random_seed 4567 \ - --default_stddev 0.046875 --max_to_keep 1 \ - --checkpoint_dir '/tmp/ckpt' \ + --n_hidden 100 --epoch $epoch_count \ + --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' \ --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/train' \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ - --lm_trie_path 'data/smoke_test/vocab.trie' \ + --lm_trie_path 'data/smoke_test/vocab.trie' diff --git a/bin/run-tc-ldc93s1_singleshotinference.sh b/bin/run-tc-ldc93s1_singleshotinference.sh index 7f6d2ba278..07a4aab174 100755 --- a/bin/run-tc-ldc93s1_singleshotinference.sh +++ b/bin/run-tc-ldc93s1_singleshotinference.sh @@ -14,19 +14,15 @@ python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ --train_files ${ldc93s1_csv} --train_batch_size 1 \ --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch 1 --random_seed 4567 --default_stddev 0.046875 \ + --n_hidden 100 --epoch 1 \ --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \ --learning_rate 0.001 --dropout_rate 0.05 \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ --lm_trie_path 'data/smoke_test/vocab.trie' -python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ - --train_files ${ldc93s1_csv} --train_batch_size 1 \ - --dev_files ${ldc93s1_csv} --dev_batch_size 1 \ - --test_files ${ldc93s1_csv} --test_batch_size 1 \ - --n_hidden 494 --epoch 1 --random_seed 4567 --default_stddev 0.046875 \ - --max_to_keep 1 --checkpoint_dir '/tmp/ckpt' --checkpoint_secs 0 \ - --learning_rate 0.001 --dropout_rate 0.05 \ +python -u DeepSpeech.py \ + --n_hidden 100 \ + --checkpoint_dir '/tmp/ckpt' \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ --lm_trie_path 'data/smoke_test/vocab.trie' \ --one_shot_infer 'data/smoke_test/LDC93S1.wav' diff --git a/bin/run-tc-ldc93s1_tflite.sh b/bin/run-tc-ldc93s1_tflite.sh index 04b0ce8296..e2e8fb61ba 100755 --- a/bin/run-tc-ldc93s1_tflite.sh +++ b/bin/run-tc-ldc93s1_tflite.sh @@ -11,10 +11,9 @@ if [ ! -f "${ldc93s1_dir}/ldc93s1.csv" ]; then fi; python -u DeepSpeech.py --noshow_progressbar \ - --n_hidden 494 \ + --n_hidden 100 \ --checkpoint_dir '/tmp/ckpt' \ - --export_dir '/tmp/train' \ + --export_dir '/tmp/train_tflite' \ --lm_binary_path 'data/smoke_test/vocab.pruned.lm' \ --lm_trie_path 'data/smoke_test/vocab.trie' \ - --notrain --notest \ - --export_tflite \ + --export_tflite --nouse_seq_length diff --git a/evaluate.py b/evaluate.py index a231029f23..e60abc09b4 100755 --- a/evaluate.py +++ b/evaluate.py @@ -5,92 +5,67 @@ import itertools import json import numpy as np -import os -import pandas import progressbar -import sys -import tables import tensorflow as tf -from collections import namedtuple from ds_ctcdecoder import ctc_beam_search_decoder_batch, Scorer -from multiprocessing import Pool, cpu_count +from multiprocessing import cpu_count from six.moves import zip, range -from util.audio import audiofile_to_input_vector from util.config import Config, initialize_globals +from util.evaluate_tools import calculate_report +from util.feeding import create_dataset from util.flags import create_flags, FLAGS from util.logging import log_error -from util.preprocess import preprocess -from util.text import Alphabet, levenshtein -from util.evaluate_tools import process_decode_result, calculate_report +from util.text import levenshtein -def split_data(dataset, batch_size): - remainder = len(dataset) % batch_size - if remainder != 0: - dataset = dataset[:-remainder] - for i in range(0, len(dataset), batch_size): - yield dataset[i:i + batch_size] +def sparse_tensor_value_to_texts(value, alphabet): + r""" + Given a :class:`tf.SparseTensor` ``value``, return an array of Python strings + representing its values, converting tokens to strings using ``alphabet``. + """ + return sparse_tuple_to_texts((value.indices, value.values, value.dense_shape), alphabet) -def pad_to_dense(jagged): - maxlen = max(len(r) for r in jagged) - subshape = jagged[0].shape +def sparse_tuple_to_texts(tuple, alphabet): + indices = tuple[0] + values = tuple[1] + results = [''] * tuple[2][0] + for i in range(len(indices)): + index = indices[i][0] + results[index] += alphabet.string_from_label(values[i]) + # List of strings + return results - padded = np.zeros((len(jagged), maxlen) + - subshape[1:], dtype=jagged[0].dtype) - for i, row in enumerate(jagged): - padded[i, :len(row)] = row - return padded - -def evaluate(test_data, inference_graph): +def evaluate(test_csvs, create_model): scorer = Scorer(FLAGS.lm_alpha, FLAGS.lm_beta, FLAGS.lm_binary_path, FLAGS.lm_trie_path, Config.alphabet) + test_set, test_batches = create_dataset(test_csvs, + batch_size=FLAGS.test_batch_size, + cache_path=FLAGS.test_cached_features_path) + it = test_set.make_one_shot_iterator() - def create_windows(features): - num_strides = len(features) - (Config.n_context * 2) + (batch_x, batch_x_len), batch_y = it.get_next() - # Create a view into the array with overlapping strides of size - # numcontext (past) + 1 (present) + numcontext (future) - window_size = 2*Config.n_context+1 - features = np.lib.stride_tricks.as_strided( - features, - (num_strides, window_size, Config.n_input), - (features.strides[0], features.strides[0], features.strides[1]), - writeable=False) + # One rate per layer + no_dropout = [None] * 6 + logits, _ = create_model(batch_x=batch_x, + seq_length=batch_x_len, + dropout=no_dropout) - return features + # Transpose to batch major and apply softmax for decoder + transposed = tf.nn.softmax(tf.transpose(logits, [1, 0, 2])) - # Create overlapping windows over the features - test_data['features'] = test_data['features'].apply(create_windows) + loss = tf.nn.ctc_loss(labels=batch_y, + inputs=logits, + sequence_length=batch_x_len) with tf.Session(config=Config.session_config) as session: - inputs, outputs, layers = inference_graph - - # Transpose to batch major for decoder - transposed = tf.transpose(outputs['outputs'], [1, 0, 2]) - - labels_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size, None], name="labels") - label_lengths_ph = tf.placeholder(tf.int32, [FLAGS.test_batch_size], name="label_lengths") - - # We add 1 to all elements of the transcript to avoid any zero values - # since we use that as an end-of-sequence token for converting the batch - # into a SparseTensor. So here we convert the placeholder back into a - # SparseTensor and subtract ones to get the real labels. - sparse_labels = tf.contrib.layers.dense_to_sparse(labels_ph) - neg_ones = tf.SparseTensor(sparse_labels.indices, -1 * tf.ones_like(sparse_labels.values), sparse_labels.dense_shape) - sparse_labels = tf.sparse_add(sparse_labels, neg_ones) - - loss = tf.nn.ctc_loss(labels=sparse_labels, - inputs=layers['raw_logits'], - sequence_length=inputs['input_lengths']) - # Create a saver using variables from the above newly created graph - mapping = {v.op.name: v for v in tf.global_variables() if not v.op.name.startswith('previous_state_')} - saver = tf.train.Saver(mapping) + saver = tf.train.Saver() # Restore variables from training checkpoint checkpoint = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) @@ -103,51 +78,38 @@ def create_windows(features): logitses = [] losses = [] + seq_lengths = [] + ground_truths = [] print('Computing acoustic model predictions...') - batch_count = len(test_data) // FLAGS.test_batch_size - bar = progressbar.ProgressBar(max_value=batch_count, + bar = progressbar.ProgressBar(max_value=test_batches, widget=progressbar.AdaptiveETA) # First pass, compute losses and transposed logits for decoding - for batch in bar(split_data(test_data, FLAGS.test_batch_size)): - session.run(outputs['initialize_state']) - - features = pad_to_dense(batch['features'].values) - features_len = batch['features_len'].values - labels = pad_to_dense(batch['transcript'].values + 1) - label_lengths = batch['transcript_len'].values - - logits, loss_ = session.run([transposed, loss], feed_dict={ - inputs['input']: features, - inputs['input_lengths']: features_len, - labels_ph: labels, - label_lengths_ph: label_lengths - }) + for batch in bar(range(test_batches)): + logits, loss_, lengths, transcripts = session.run([transposed, loss, batch_x_len, batch_y]) logitses.append(logits) losses.extend(loss_) + seq_lengths.append(lengths) + ground_truths.extend(sparse_tensor_value_to_texts(transcripts, Config.alphabet)) - ground_truths = [] predictions = [] - print('Decoding predictions...') - bar = progressbar.ProgressBar(max_value=batch_count, - widget=progressbar.AdaptiveETA) - # Get number of accessible CPU cores for this process try: num_processes = cpu_count() except: num_processes = 1 + print('Decoding predictions...') + bar = progressbar.ProgressBar(max_value=test_batches, + widget=progressbar.AdaptiveETA) + # Second pass, decode logits and compute WER and edit distance metrics - for logits, batch in bar(zip(logitses, split_data(test_data, FLAGS.test_batch_size))): - seq_lengths = batch['features_len'].values.astype(np.int32) - decoded = ctc_beam_search_decoder_batch(logits, seq_lengths, Config.alphabet, FLAGS.beam_width, + for logits, seq_length in bar(zip(logitses, seq_lengths)): + decoded = ctc_beam_search_decoder_batch(logits, seq_length, Config.alphabet, FLAGS.beam_width, num_processes=num_processes, scorer=scorer) - - ground_truths.extend(Config.alphabet.decode(l) for l in batch['transcript']) predictions.extend(d[0][1] for d in decoded) distances = [levenshtein(a, b) for a, b in zip(ground_truths, predictions)] @@ -179,21 +141,8 @@ def main(_): 'the --test_files flag.') exit(1) - # sort examples by length, improves packing of batches and timesteps - test_data = preprocess( - FLAGS.test_files.split(','), - FLAGS.test_batch_size, - alphabet=Config.alphabet, - numcep=Config.n_input, - numcontext=Config.n_context, - hdf5_cache_path=FLAGS.hdf5_test_set).sort_values( - by="features_len", - ascending=False) - - from DeepSpeech import create_inference_graph - graph = create_inference_graph(batch_size=FLAGS.test_batch_size, n_steps=-1) - - samples = evaluate(test_data, graph) + from DeepSpeech import create_model + samples = evaluate(FLAGS.test_files.split(','), create_model) if FLAGS.test_output_file: # Save decoded tuples as JSON, converting NumPy floats to Python floats diff --git a/native_client/BUILD b/native_client/BUILD index 1e5587deda..bf4e1d2654 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -21,6 +21,14 @@ genrule( local = 1, ) +genrule( + name = "ds_graph_version", + outs = ["ds_graph_version.h"], + cmd = "$(location :ds_graph_version.sh) >$@", + tools = [":ds_graph_version.sh"], + local = 1, +) + KENLM_SOURCES = glob(["kenlm/lm/*.cc", "kenlm/util/*.cc", "kenlm/util/double-conversion/*.cc", "kenlm/lm/*.hh", "kenlm/util/*.hh", "kenlm/util/double-conversion/*.h"], exclude = ["kenlm/*/*test.cc", "kenlm/*/*main.cc"]) @@ -62,15 +70,8 @@ tf_cc_shared_object( srcs = ["deepspeech.cc", "deepspeech.h", "alphabet.h", - "c_speech_features/c_speech_features.cpp", - "kiss_fft130/kiss_fft.c", - "kiss_fft130/tools/kiss_fftr.c", - "c_speech_features/c_speech_features.h", - "c_speech_features/c_speech_features_config.h", - "kiss_fft130/kiss_fft.h", - "kiss_fft130/_kiss_fft_guts.h", - "kiss_fft130/tools/kiss_fftr.h", - "ds_version.h"] + + "ds_version.h", + "ds_graph_version.h"] + DECODER_SOURCES, copts = select({ # -fvisibility=hidden is not required on Windows, MSCV hides all declarations by default @@ -119,6 +120,10 @@ tf_cc_shared_object( "//tensorflow/core/kernels:control_flow_ops", # Enter "//tensorflow/core/kernels:tile_ops", # Tile "//tensorflow/core/kernels:gather_op", # Gather + "//tensorflow/core/kernels:mfcc_op", # Mfcc + "//tensorflow/core/kernels:spectrogram_op", # AudioSpectrogram + "//tensorflow/core/kernels:strided_slice_op", # StridedSlice + "//tensorflow/core/kernels:slice_op", # Slice, needed by StridedSlice "//tensorflow/contrib/rnn:lstm_ops_kernels", # BlockLSTM "//tensorflow/core/kernels:random_ops", # RandomGammaGrad "//tensorflow/core/kernels:pack_op", # Pack @@ -130,7 +135,7 @@ tf_cc_shared_object( }) + if_cuda([ "//tensorflow/core:core", ]), - includes = ["c_speech_features", "kiss_fft130"] + DECODER_INCLUDES, + includes = DECODER_INCLUDES, defines = ["KENLM_MAX_ORDER=6"], ) diff --git a/native_client/c_speech_features/c_speech_features.cpp b/native_client/c_speech_features/c_speech_features.cpp deleted file mode 100644 index fc1de14235..0000000000 --- a/native_client/c_speech_features/c_speech_features.cpp +++ /dev/null @@ -1,588 +0,0 @@ -#ifdef _MSC_VER - #define _USE_MATH_DEFINES - #include -#else /*_MSC_VER*/ - #include -#endif - - -#include "c_speech_features.h" -#include "tools/kiss_fftr.h" - -#define MAX(x,y) ((x) > (y) ? (x) : (y)) -#define MIN(x,y) ((x) < (y) ? (x) : (y)) -#define CLAMP(x,y,z) MIN(MAX(x,y),z) - -template -int -csf_mfcc(const T* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNCep, int aNFilters, - int aNFFT, int aLowFreq, int aHighFreq, csf_float aPreemph, - int aCepLifter, int aAppendEnergy, csf_float* aWinFunc, - csf_float** aMFCC) -{ - int i, j, k, idx, fidx, didx; - csf_float* feat; - csf_float* energy; - - int n_frames = csf_logfbank(aSignal, aSignalLen, aSampleRate, aWinLen, aWinStep, - aNFilters, aNFFT, aLowFreq, aHighFreq, aPreemph, - aWinFunc, &feat, aAppendEnergy ? &energy : NULL); - - // Allocate an array so we can calculate the inner loop multipliers - // in the DCT-II just one time. - double* dct2f = (double*)malloc(sizeof(double) * aNFilters * aNCep); - - // Perform DCT-II - double sf1 = csf_sqrt(1 / (4 * (double)aNFilters)); - double sf2 = csf_sqrt(1 / (2 * (double)aNFilters)); - csf_float* mfcc = (csf_float*)malloc(sizeof(csf_float) * n_frames * aNCep); - for (i = 0, idx = 0, fidx = 0; i < n_frames; - i++, idx += aNCep, fidx += aNFilters) { - for (j = 0, didx = 0; j < aNCep; j++) { - double sum = 0.0; - for (k = 0; k < aNFilters; k++, didx++) { - if (i == 0) { - dct2f[didx] = cos(M_PI * j * (2 * k + 1) / (double)(2 * aNFilters)); - } - sum += (double)feat[fidx+k] * dct2f[didx]; - } - mfcc[idx+j] = (csf_float)(sum * 2.0 * ((i == 0 && j == 0) ? sf1 : sf2)); - } - } - - // Free inner-loop multiplier cache - free(dct2f); - - // Free features array - free(feat); - - // Apply a cepstral lifter - if (aCepLifter != 0) { - csf_lifter(mfcc, n_frames, aNCep, aCepLifter); - } - - // Append energies - if (aAppendEnergy) { - for (i = 0, idx = 0; i < n_frames; i++, idx += aNCep) { - mfcc[idx] = csf_log(energy[i]); - } - - // Free energy array - free(energy); - } - - // Return MFCC features - *aMFCC = mfcc; - - return n_frames; -} - -template -int -csf_mfcc(const short* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNCep, int aNFilters, - int aNFFT, int aLowFreq, int aHighFreq, csf_float aPreemph, - int aCepLifter, int aAppendEnergy, csf_float* aWinFunc, - csf_float** aMFCC); - -template -int -csf_mfcc(const float* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNCep, int aNFilters, - int aNFFT, int aLowFreq, int aHighFreq, csf_float aPreemph, - int aCepLifter, int aAppendEnergy, csf_float* aWinFunc, - csf_float** aMFCC); - - -template -int -csf_fbank(const T* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures, csf_float** aEnergy) -{ - int i, j, k, idx, fidx, pidx; - csf_float* feat; - csf_float* fbank; - csf_float* pspec; - csf_float* frames; - csf_float* energy; - csf_float* preemph = csf_preemphasis(aSignal, aSignalLen, aPreemph); - int frame_len = (int)round(aWinLen * aSampleRate); - int frame_step = (int)round(aWinStep * aSampleRate); - int feat_width = aNFFT / 2 + 1; - - // Frame the signal into overlapping frames - int n_frames = csf_framesig(preemph, aSignalLen, frame_len, aNFFT, - frame_step, aWinFunc, &frames); - - // Free preemphasised signal buffer - free(preemph); - - // Compute the power spectrum of the frames - pspec = csf_powspec((const csf_float*)frames, n_frames, aNFFT); - - // Free frames - free(frames); - - // Store the total energy in each frame - if (aEnergy) { - energy = (csf_float*)calloc(n_frames, sizeof(csf_float)); - for (i = 0, idx = 0; i < n_frames; i++) { - for (j = 0; j < feat_width; j++, idx++) { - energy[i] += pspec[idx]; - } - if (energy[i] == 0.0) { - energy[i] = csf_float_min; - } - } - } - - // Compute the filter-bank energies - fbank = csf_get_filterbanks(aNFilters, aNFFT, aSampleRate, - aLowFreq, aHighFreq); - feat = (csf_float*)calloc(n_frames * aNFilters, sizeof(csf_float)); - for (i = 0, idx = 0, pidx = 0; i < n_frames; - i++, idx += aNFilters, pidx += feat_width) { - for (j = 0, fidx = 0; j < aNFilters; j++) { - for (k = 0; k < feat_width; k++, fidx++) { - feat[idx + j] += pspec[pidx + k] * fbank[fidx]; - } - if (feat[idx + j] == 0.0) { - feat[idx + j] = csf_float_min; - } - } - } - - // Free fbank - free(fbank); - - // Free pspec - free(pspec); - - // Return features and energies - *aFeatures = feat; - if (aEnergy) { - *aEnergy = energy; - } - - return n_frames; -} - -template -int -csf_fbank(const short* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures, csf_float** aEnergy); - -template -int -csf_fbank(const float* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures, csf_float** aEnergy); - -template -int -csf_logfbank(const T* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, - csf_float* aWinFunc, csf_float** aFeatures, csf_float** aEnergy) -{ - int i, j, idx; - int n_frames = csf_fbank(aSignal, aSignalLen, aSampleRate, aWinLen, aWinStep, - aNFilters, aNFFT, aLowFreq, aHighFreq, aPreemph, - aWinFunc, aFeatures, aEnergy); - - for (i = 0, idx = 0; i < n_frames; i++) { - for (j = 0; j < aNFilters; j++, idx++) { - (*aFeatures)[idx] = csf_log((*aFeatures)[idx]); - } - } - - return n_frames; -} - -template -int -csf_logfbank(const short* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, - csf_float* aWinFunc, csf_float** aFeatures, csf_float** aEnergy); - -template -int -csf_logfbank(const float* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, - csf_float* aWinFunc, csf_float** aFeatures, csf_float** aEnergy); - -template -int -csf_ssc(const T* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures) -{ - int i, j, k, idx, pidx, fidx; - csf_float* ssc; - csf_float* feat; - csf_float* fbank; - csf_float* pspec; - csf_float* frames; - csf_float* preemph = csf_preemphasis(aSignal, aSignalLen, aPreemph); - int frame_len = (int)round(aWinLen * aSampleRate); - int frame_step = (int)round(aWinStep * aSampleRate); - int feat_width = aNFFT / 2 + 1; - - // Frame the signal into overlapping frames - int n_frames = csf_framesig(preemph, aSignalLen, frame_len, aNFFT, - frame_step, aWinFunc, &frames); - - // Free preemphasised signal buffer - free(preemph); - - // Compute the power spectrum of the frames - pspec = csf_powspec((const csf_float*)frames, n_frames, aNFFT); - - // Free frames - free(frames); - - // Make sure there are no zeroes in the power spectrum - for (i = 0, idx = 0; i < n_frames; i++) { - for (j = 0; j < feat_width; j++, idx++) { - if (pspec[idx] == 0.0) { - pspec[idx] = csf_float_min; - } - } - } - - // Compute the filter-bank energies - fbank = csf_get_filterbanks(aNFilters, aNFFT, aSampleRate, - aLowFreq, aHighFreq); - feat = (csf_float*)calloc(n_frames * aNFilters, sizeof(csf_float)); - for (i = 0, idx = 0, pidx = 0; i < n_frames; - i++, idx += aNFilters, pidx += feat_width) { - for (j = 0, fidx = 0; j < aNFilters; j++) { - for (k = 0; k < feat_width; k++, fidx++) { - feat[idx + j] += pspec[pidx + k] * fbank[fidx]; - } - } - } - - // Calculate Spectral Sub-band Centroid features - ssc = (csf_float*)calloc(n_frames * aNFilters, sizeof(csf_float)); - csf_float r = ((aSampleRate / 2) - 1) / (csf_float)(feat_width - 1); - for (i = 0, idx = 0, pidx = 0; i < n_frames; - i++, idx += aNFilters, pidx += feat_width) { - for (j = 0, fidx = 0; j < aNFilters; j++) { - csf_float R = 1; - for (k = 0; k < feat_width; k++, fidx++) { - ssc[idx + j] += pspec[pidx + k] * R * fbank[fidx]; - R += r; - } - ssc[idx + j] /= feat[idx + j]; - } - } - - // Free arrays we've finished with - free(fbank); - free(pspec); - free(feat); - - // Return features - *aFeatures = ssc; - - return n_frames; -} - -template -int -csf_ssc(const short* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures); - -template -int -csf_ssc(const float* aSignal, unsigned int aSignalLen, int aSampleRate, - csf_float aWinLen, csf_float aWinStep, int aNFilters, int aNFFT, - int aLowFreq, int aHighFreq, csf_float aPreemph, csf_float* aWinFunc, - csf_float** aFeatures); - -csf_float -csf_hz2mel(csf_float aHz) -{ - return CSF_HZ2MEL(aHz); -} - -csf_float -csf_mel2hz(csf_float aMel) -{ - return CSF_MEL2HZ(aMel); -} - -void -csf_lifter(csf_float* aCepstra, int aNFrames, int aNCep, int aCepLifter) -{ - int i, j, idx; - - csf_float lifter = aCepLifter / 2.0; - csf_float* factors = (csf_float*)malloc(sizeof(csf_float) * aNCep); - for (i = 0; i < aNCep; i++) { - factors[i] = 1 + lifter * csf_sin(M_PI * i / (csf_float)aCepLifter); - } - - for (i = 0, idx = 0; i < aNFrames; i++) { - for (j = 0; j < aNCep; j++, idx++) { - aCepstra[idx] *= factors[j]; - } - } - - free(factors); -} - -csf_float* -csf_delta(const csf_float* aFeatures, int aNFrames, int aNFrameLen, int aN) -{ - int i, j, k, idx; - csf_float* delta; - - if (aN < 1) { - return NULL; - } - - csf_float denominator = 0; - for (i = 1; i <= aN; i++) { - denominator += csf_pow(i, 2); - } - denominator *= 2; - - delta = (csf_float*)calloc(aNFrames * aNFrameLen, sizeof(csf_float)); - for (i = 0, idx = 0; i < aNFrames; i++, idx += aNFrameLen) { - for (j = 0; j < aNFrameLen; j++) { - for (k = -aN; k <= aN; k++) { - delta[idx + j] += k * - CSF_2D_REF(aFeatures, aNFrameLen, j, CLAMP(i + k, 0, aNFrames - 1)); - } - delta[idx + j] /= denominator; - } - } - - return delta; -} - -csf_float* -csf_get_filterbanks(int aNFilters, int aNFFT, int aSampleRate, - int aLowFreq, int aHighFreq) -{ - int i, j, idx; - int feat_width = aNFFT / 2 + 1; - csf_float lowmel = CSF_HZ2MEL(aLowFreq); - csf_float highmel = CSF_HZ2MEL((aHighFreq <= aLowFreq) ? - aSampleRate / 2 : aHighFreq); - int* bin = (int*)malloc(sizeof(int) * (aNFilters + 2)); - csf_float* fbank = - (csf_float*)calloc(aNFilters * feat_width, sizeof(csf_float)); - - for (i = 0; i < aNFilters + 2; i++) { - csf_float melpoint = ((highmel - lowmel) / - (csf_float)(aNFilters + 1) * i) + lowmel; - bin[i] = (int)csf_floor((aNFFT + 1) * - CSF_MEL2HZ(melpoint) / (csf_float)aSampleRate); - } - - for (i = 0, idx = 0; i < aNFilters; i++, idx += feat_width) { - int start = MIN(bin[i], bin[i+1]); - int end = MAX(bin[i], bin[i+1]); - for (j = start; j < end; j++) { - fbank[idx + j] = (j - bin[i]) / (csf_float)(bin[i+1]-bin[i]); - } - start = MIN(bin[i+1], bin[i+2]); - end = MAX(bin[i+1], bin[i+2]); - for (j = start; j < end; j++) { - fbank[idx + j] = (bin[i+2]-j) / (csf_float)(bin[i+2]-bin[i+1]); - } - } - free(bin); - - return fbank; -} - -int -csf_framesig(const csf_float* aSignal, unsigned int aSignalLen, int aFrameLen, - int aPaddedFrameLen, int aFrameStep, csf_float* aWinFunc, - csf_float** aFrames) -{ - int* indices; - csf_float* frames; - int i, j, idx, iidx, n_frames; - int frame_width = MAX(aPaddedFrameLen, aFrameLen); - - if (aSignalLen > aFrameLen) { - n_frames = 1 + (int)csf_ceil((aSignalLen - aFrameLen) / - (csf_float)aFrameStep); - } else { - n_frames = 1; - } - - indices = (int*)malloc(sizeof(int) * n_frames * aFrameLen); - for (i = 0, idx = 0; i < n_frames; i++) { - int base = i * aFrameStep; - for (j = 0; j < aFrameLen; j++, idx++) { - indices[idx] = base + j; - } - } - - frames = (csf_float*)malloc(sizeof(csf_float) * n_frames * frame_width); - for (i = 0, idx = 0, iidx = 0; i < n_frames; i++) { - for (j = 0; j < aFrameLen; j++, idx++, iidx++) { - int index = indices[iidx]; - frames[idx] = index < aSignalLen ? aSignal[index] : 0.0; - if (aWinFunc) { - frames[idx] *= aWinFunc[j]; - } - } - for (j = aFrameLen; j < aPaddedFrameLen; j++, idx++) { - frames[idx] = 0.0; - } - } - free(indices); - - *aFrames = frames; - return n_frames; -} - -int -csf_deframesig(const csf_float* aFrames, int aNFrames, int aSigLen, - int aFrameLen, int aFrameStep, csf_float* aWinFunc, - csf_float** aSignal) -{ - int i, j, base, idx; - csf_float* signal; - csf_float* win_correct; - int padlen = (aNFrames - 1) * aFrameStep + aFrameLen; - - if (aSigLen <= 0) { - aSigLen = padlen; - } - - win_correct = (csf_float*)calloc(aSigLen, sizeof(csf_float)); - - base = 0; - signal = (csf_float*)calloc(aSigLen, sizeof(csf_float)); - for (i = 0, idx = 0; i < aNFrames; i++) { - for (j = 0; j < aFrameLen; j++, idx++) { - int sidx = j + base; - if (sidx >= aSigLen) { - continue; - } - signal[sidx] += aFrames[idx]; - if (aWinFunc) { - win_correct[sidx] += aWinFunc[j] + 1e-15; - } else { - win_correct[sidx] += 1 + 1e-15; - } - } - base += aFrameStep; - } - - for (i = 0; i < aSigLen; i++) { - signal[i] /= win_correct[i]; - } - free(win_correct); - - *aSignal = signal; - return aSigLen; -} - -template -csf_float* -csf_preemphasis(const T* aSignal, unsigned int aSignalLen, csf_float aCoeff) -{ - int i; - csf_float* preemph = (csf_float*)malloc(sizeof(csf_float) * aSignalLen); - - for (i = aSignalLen - 1; i >= 1; i--) { - preemph[i] = aSignal[i] - aSignal[i-1] * aCoeff; - } - preemph[0] = (csf_float)aSignal[0]; - - return preemph; -} - -template -csf_float* -csf_preemphasis(const short* aSignal, unsigned int aSignalLen, csf_float aCoeff); - -template -csf_float* -csf_preemphasis(const float* aSignal, unsigned int aSignalLen, csf_float aCoeff); - -csf_float* -csf_magspec(const csf_float* aFrames, int aNFrames, int aNFFT) -{ - int i, j, idx; - const int fft_out = aNFFT / 2 + 1; - kiss_fftr_cfg cfg = kiss_fftr_alloc(aNFFT, 0, NULL, NULL); - csf_float* mspec = (csf_float*)malloc(sizeof(csf_float) * aNFrames * fft_out); - kiss_fft_cpx* out = (kiss_fft_cpx*)malloc(sizeof(kiss_fft_cpx) * fft_out); - - for (i = 0, idx = 0; i < aNFrames; i++) { - // Compute the magnitude spectrum - kiss_fftr(cfg, &(aFrames[i * aNFFT]), out); - for (j = 0; j < fft_out; j++, idx++) { - mspec[idx] = csf_sqrt(csf_pow(out[j].r, 2.0) + csf_pow(out[j].i, 2.0)); - } - } - - KISS_FFT_FREE(cfg); - free(out); - return mspec; -} - -csf_float* -csf_powspec(const csf_float* aFrames, int aNFrames, int aNFFT) -{ - int i; - const int fft_out = aNFFT / 2 + 1; - csf_float* pspec = csf_magspec(aFrames, aNFrames, aNFFT); - - // Compute the power spectrum - for (i = 0; i < aNFrames * fft_out; i++) { - pspec[i] = (1.0/aNFFT) * powf(pspec[i], 2.0); - } - - return pspec; -} - -csf_float* -csf_logpowspec(const csf_float* aFrames, int aNFrames, int aNFFT, int aNorm) -{ - int i; - const int frames_len = aNFrames * (aNFFT / 2 + 1); - - csf_float* logpspec = csf_powspec(aFrames, aNFrames, aNFFT); - - csf_float max = 0; - for (i = 0; i < frames_len; i++) { - if (logpspec[i] < 1e-30f) { - logpspec[i] = -300; - } else { - logpspec[i] = 10.0 * csf_log10(logpspec[i]); - } - if (aNorm && logpspec[i] > max) { - max = logpspec[i]; - } - } - - if (aNorm) { - for (i = 0; i < frames_len; i++) { - logpspec[i] -= max; - } - } - - return logpspec; -} diff --git a/native_client/c_speech_features/c_speech_features.h b/native_client/c_speech_features/c_speech_features.h deleted file mode 100644 index 2e01f43564..0000000000 --- a/native_client/c_speech_features/c_speech_features.h +++ /dev/null @@ -1,410 +0,0 @@ - -/** - * Calculate filterbank features. Provides e.g. fbank and mfcc features for use - * in ASR applications. - * - * Derived from python_speech_features, by James Lyons. - * Port by Chris Lord. - */ - -#ifndef __C_SPEECH_FEATURES_H__ -#define __C_SPEECH_FEATURES_H__ - -#include -#include "c_speech_features_config.h" - -#define CSF_HZ2MEL(x) (2595.0 * csf_log10(1.0+(x)/700.0)) -#define CSF_MEL2HZ(x) (700.0 * (csf_pow(10.0, (x)/2595.0) - 1.0)) - -#define CSF_2D_INDEX(w,x,y) (((y)*(w))+(x)) -#define CSF_2D_REF(m,w,x,y) ((m)[CSF_2D_INDEX(w,x,y)]) - -/** - * @brief Compute MFCC features from an audio signal. - * - * @param aSignal The audio signal from which to compute features. - * @param aSignalLen The length of the audio signal array. - * @param aSampleRate The sample-rate of the signal. - * @param aWinLen The length of the analysis window in seconds. (e.g. 0.025) - * @param aWinStep The step between successive windows in seconds. (e.g. 0.01) - * @param aNCep The number of cepstrum to return. (e.g. 13) - * @param aNFilters The number of filters in the filterbank. (e.g. 26) - * @param aNFFT The FFT size. (e.g. 512) - * @param aLowFreq The lowest band edge of mel filters, in hz. (e.g. 0) - * @param aHighFreq The highest band edge of mel filters, in hz. Must not be - * higher than @p aSampleRate / 2. If this is lower or equal - * to @p aLowFreq, it will be treated as @p aSampleRate / 2. - * @param aPreemph Preemphasis filter coefficient. 0 is no filter. (e.g. 0.97) - * @param aCepLifter The lifting coefficient to use. 0 disables lifting. - * (e.g. 22) - * @param aAppendEnergy If this is true, the zeroth cepstral coefficient is - * replaced with the log of the total frame energy. - * @param aWinFunc An array of size @c frameLen, as determined by multiplying - * @p aWinLen by @p aSmapleRate, or @c NULL to be used as an - * analysis window to apply to each frame. Refer to - * csf_framesig(). - * @param[out] aMFCC An array containing features, of shape - * (frames, @p aNCep). The user is responsible for freeing - * the array. - * - * @return The number of frames. - */ -template -int csf_mfcc(const T* aSignal, - unsigned int aSignalLen, - int aSampleRate, - csf_float aWinLen, - csf_float aWinStep, - int aNCep, - int aNFilters, - int aNFFT, - int aLowFreq, - int aHighFreq, - csf_float aPreemph, - int aCepLifter, - int aAppendEnergy, - csf_float* aWinFunc, - csf_float** aMFCC); - -/** - * @brief Compute Mel-filterbank energy features from an audio signal. - * - * Compute Mel-filterbank energy features from an audio signal. - * - * @param aSignal The audio signal from which to compute features. - * @param aSignalLen The length of the audio signal array. - * @param aSampleRate The sample-rate of the signal. - * @param aWinLen The length of the analysis window in seconds. (e.g. 0.025) - * @param aWinStep The step between successive windows in seconds. (e.g. 0.01) - * @param aNFilters The number of filters in the filterbank. (e.g. 26) - * @param aNFFT The FFT size. (e.g. 512) - * @param aLowFreq The lowest band edge of mel filters, in hz. (e.g. 0) - * @param aHighFreq The highest band edge of mel filters, in hz. Must not be - * higher than @p aSampleRate / 2. If this is lower or equal - * to @p aLowFreq, it will be treated as @p aSampleRate / 2. - * @param aPreemph Preemphasis filter coefficient. 0 is no filter. (e.g. 0.97) - * @param aWinFunc An array of size @c frameLen, as determined by multiplying - * @p aWinLen by @p aSmapleRate, or @c NULL to be used as an - * analysis window to apply to each frame. Refer to - * csf_framesig(). - * @param[out] aFeatures A 2D array containing features, of shape - * (frames, @p aNFilters). The user is responsible for - * freeing the array. - * @param[out] aEnergy An array containing energies, of shape (frames), or - * @c NULL. The user is responsible for freeing the array. - * - * @return The number of frames. - */ -template -int csf_fbank(const T* aSignal, - unsigned int aSignalLen, - int aSampleRate, - csf_float aWinLen, - csf_float aWinStep, - int aNFilters, - int aNFFT, - int aLowFreq, - int aHighFreq, - csf_float aPreemph, - csf_float* aWinFunc, - csf_float** aFeatures, - csf_float** aEnergy); - -/** - * @brief Compute log Mel-filterbank energy features from an audio signal. - * - * Compute log Mel-filterbank energy features from an audio signal. - * - * @param aSignal The audio signal from which to compute features. - * @param aSignalLen The length of the audio signal array. - * @param aSampleRate The sample-rate of the signal. - * @param aWinLen The length of the analysis window in seconds. (e.g. 0.025) - * @param aWinStep The step between successive windows in seconds. (e.g. 0.01) - * @param aNFilters The number of filters in the filterbank. (e.g. 26) - * @param aNFFT The FFT size. (e.g. 512) - * @param aLowFreq The lowest band edge of mel filters, in hz. (e.g. 0) - * @param aHighFreq The highest band edge of mel filters, in hz. Must not be - * higher than @p aSampleRate / 2. If this is lower or equal - * to @p aLowFreq, it will be treated as @p aSampleRate / 2. - * @param aPreemph Preemphasis filter coefficient. 0 is no filter. (e.g. 0.97) - * @param aWinFunc An array of size @c frameLen, as determined by multiplying - * @p aWinLen by @p aSmapleRate, or @c NULL to be used as an - * analysis window to apply to each frame. Refer to - * csf_framesig(). - * @param[out] aFeatures A 2D array containing features, of shape - * (frames, @p aNFilters). The user is responsible for - * freeing the array. - * @param[out] aEnergy An array containing energies, of shape (frames). The - * user is responsible for freeing the array. - * - * @return The number of frames. - */ -template -int csf_logfbank(const T* aSignal, - unsigned int aSignalLen, - int aSampleRate, - csf_float aWinLen, - csf_float aWinStep, - int aNFilters, - int aNFFT, - int aLowFreq, - int aHighFreq, - csf_float aPreemph, - csf_float* aWinFunc, - csf_float** aFeatures, - csf_float** aEnergy); - -/** - * @brief Compute Spectral Sub-band Centroid features from an audio signal. - * - * Compute Spectral Sub-band Centroid features from an audio signal. - * - * @param aSignal The audio signal from which to compute features. - * @param aSignalLen The length of the audio signal array. - * @param aSampleRate The sample-rate of the signal. - * @param aWinLen The length of the analysis window in seconds. (e.g. 0.025) - * @param aWinStep The step between successive windows in seconds. (e.g. 0.01) - * @param aNFilters The number of filters in the filterbank. (e.g. 26) - * @param aNFFT The FFT size. (e.g. 512) - * @param aLowFreq The lowest band edge of mel filters, in hz. (e.g. 0) - * @param aHighFreq The highest band edge of mel filters, in hz. Must not be - * higher than @p aSampleRate / 2. If this is lower or equal - * to @p aLowFreq, it will be treated as @p aSampleRate / 2. - * @param aPreemph Preemphasis filter coefficient. 0 is no filter. (e.g. 0.97) - * @param aWinFunc An array of size @c frameLen, as determined by multiplying - * @p aWinLen by @p aSmapleRate, or @c NULL to be used as an - * analysis window to apply to each frame. Refer to - * csf_framesig(). - * @param[out] aFeatures A 2D array containing features, of shape - * (frames, @p aNFilters). The user is responsible for - * freeing the array. - */ -template -int csf_ssc(const T* aSignal, - unsigned int aSignalLen, - int aSampleRate, - csf_float aWinLen, - csf_float aWinStep, - int aNFilters, - int aNFFT, - int aLowFreq, - int aHighFreq, - csf_float aPreemph, - csf_float* aWinFunc, - csf_float** aFeatures); - -/** - * @brief Convert a value in Hertz to Mels - * - * Convert a value in Hertz to Mels - * - * @param aHz A value in Hz. - * - * @return A value in Mels. - */ -csf_float csf_hz2mel(csf_float aHz); - -/** - * @brief Convert a value in Mels to Hertz - * - * Convert a value in Mels to Hertz - * - * @param aMel A value in Mels. - * - * @return A value in Hz. - */ -csf_float csf_mel2hz(csf_float aMel); - -/** - * @brief Compute a Mel-filterbank. - * - * Compute a Mel-filterbank. The filters are stored in the rows, the columns - * correspond to fft bins. The filters are returned as an array of size - * @p aNFilters * (@p aNFFT / 2 + 1). - * - * @param aNFilters The number of filters in the filterbank. (e.g. 20) - * @param aNFFT The FFT size. (e.g. 512) - * @param aSampleRate The sample-rate of the signal being worked with. Affects - * mel spacing. - * @param aLowFreq The lowest band edge of mel filters, in hz. (e.g. 0) - * @param aHighFreq The highest band edge of mel filters, in hz. Must not be - * higher than @p aSampleRate / 2. If this is lower or equal - * to @p aLowFreq, it will be treated as @p aSampleRate / 2. - * - * @return A 2D array of shape (@p aNFilters, @p aNFFT / 2 + 1). The user is - * responsible for freeing the array. - */ -csf_float* csf_get_filterbanks(int aNFilters, - int aNFFT, - int aSampleRate, - int aLowFreq, - int aHighFreq); - -/** - * @brief Apply a cepstral lifter on a matrix of cepstra. - * - * Apply a cepstral lifter on a matrix of cepstra. This has the effect of - * increasing the magnitude of high-frequency DCT coefficients. - * - * @param aCepstra The 2D array matrix of mel-cepstra. - * @param aNFrames The number of frames. - * @param aNCep The number of cepstra per frame. - * @param aCepLifter The lifting coefficient to use. 0 disables lifting. - * (e.g. 22) - */ -void csf_lifter(csf_float* aCepstra, - int aNFrames, - int aNCep, - int aCepLifter); - -/** - * @brief Compute delta features from a feature vector sequence. - * - * Compute delta features from a feature vector sequence. - * - * @param aFeatures A 2D array of shape (@p aNFeatures, @p aNFrames). Each row - * holds one feature vector. - * @param aNFrames The number of frames in @p aFeatures. - * @param aNFrameLen The length of each frame in @p aFeatures. - * @param @aN For each frame, calculate delta features based on preceding and - * following N frames. Must be 1 or larger. - * - * @return A 2D array of shape (@p aNFeatures, @p aNFrames) containing delta - * features. Each row contains holds 1 delta feature vector. The user - * is responsible for freeing the array. - */ -csf_float* csf_delta(const csf_float* aFeatures, - int aNFrames, - int aNFrameLen, - int aN); - -/** - * @brief Perform preemphasis on an input signal. - * - * Perform preemphasis on an input signal. - * - * @param aSignal The signal to filter. - * @param aSignalLen The length of the signal array. - * @param aCoeff The preemphasis coefficient. 0 is no filter. (e.g. 0.95) - * - * @return The filtered signal. The user is responsible for freeing this array. - */ -template -csf_float* csf_preemphasis(const T* aSignal, - unsigned int aSignalLen, - csf_float aCoeff); - -/** - * @brief Frame a signal into overlapping frames. - * - * Frame a signal into overlapping frames. - * - * @param aSignal The signal to frame. - * @param aSignalLen The length of the signal array. - * @param aFrameLen The length of each frame in samples. - * @param aPaddedFrameLen If greater than @p aFrameLen, @p aPaddedFrameLen - - * @p aFrameLen zeros will be appended to each frame. - * @param aFrameStep The number of samples after the start of the previous frame - * that the next frame should begin. - * @param aWinFunc An array of size @p aFrameLen, or @c NULL to be used as an - * analysis window to apply to each frame. When specified, - * each overlapping frame of the signal will be multiplied - * by the value in the corresponding index of the array. - * @param[out] aFrames A 2D array of frames, of shape - * (@c frames, @p aPaddedFrameLen). - * The user is responsible for freeing the array. - * - * @return The number of frames. - */ -int csf_framesig(const csf_float* aSignal, - unsigned int aSignalLen, - int aFrameLen, - int aPaddedFrameLen, - int aFrameStep, - csf_float* aWinFunc, - csf_float** aFrames); - -/** - * @brief Perform overlap-add procedure to undo the action of csf_framesig(). - * - * Perform overlap-add procedure to undo the action of csf_framesig(). - * - * @param aFrames The 2D array of frames. - * @param aNFrames The number of frames in @p aFrames. - * @param aSigLen The length of the desired signal, or 0 if unknown. - * @param aFrameLen The length of each frame in samples. - * @param aFrameStep The number of samples after the start of the previous frame - * that the next frame begins - * @param aWinFunc An array of size @p aFrameLen, or @c NULL to be used as an - * analysis window to apply to each frame. When specified, - * each sample of the signal will be divided by the aggregated - * value in the corresponding indices of the array. - * @param[out] aSignal An array of samples. The length will be @p aSigLen if - * specified. The user is responsible for freeing - * this array. - * - * @return Returns the length of @p aSignal. - */ -int csf_deframesig(const csf_float* aFrames, - int aNFrames, - int aSigLen, - int aFrameLen, - int aFrameStep, - csf_float* aWinFunc, - csf_float** aSignal); - -/** - * @brief Compute the magnitude spectrum of frames. - * - * Compute the magnitude spectrum of each frame in frames. - * - * @param aFrames The 2D array of frames, of shape (@p aNFrames, @p aNFFT). - * @param aNFrames The number of frames. - * @param aNFFT The FFT length to use. - * - * @return A 2D array containing the magnitude spectrum of the - * corresponding frame, of shape (@p aNFrames, @p aNFFT / 2 + 1). The - * user is responsible for freeing the array. - */ -csf_float* csf_magspec(const csf_float* aFrames, - int aNFrames, - int aNFFT); - -/** - * @brief Compute the power spectrum of frames. - * - * Compute the power spectrum of each frame in frames. - * - * @param aFrames The 2D array of frames, of shape (@p aNFrames, @p aNFFT). - * @param aNFrames The number of frames. - * @param aNFFT The FFT length to use. - * - * @return A 2D array containing the power spectrum of the - * corresponding frame, of shape (@p aNFrames, @p aNFFT / 2 + 1). - * The user is responsible for freeing the array. - */ -csf_float* csf_powspec(const csf_float* aFrames, - int aNFrames, - int aNFFT); - -/** - * @brief Compute the log power spectrum of frames. - * - * Compute the log power spectrum of each frame in frames. - * - * @param aFrames The 2D array of frames, of shape (@p aNFrames, @p aNFFT). - * @param aNFrames The number of frames. - * @param aNFFT The FFT length to use. - * @param aNorm If not zero, the log power spectrum is normalised so that the - * maximum value across all frames is 0. - * - * @return A 2D array containing the log power spectrum of the - * corresponding frame, of shape (@p aNFrames, @p aNFFT / 2 + 1). - * The user is responsible for freeing the array. - */ -csf_float* csf_logpowspec(const csf_float* aFrames, - int aNFrames, - int aNFFT, - int aNorm); - -#endif /* __C_SPEECH_FEATURES_H__ */ diff --git a/native_client/c_speech_features/c_speech_features_config.h b/native_client/c_speech_features/c_speech_features_config.h deleted file mode 100644 index bff73dfc6d..0000000000 --- a/native_client/c_speech_features/c_speech_features_config.h +++ /dev/null @@ -1,26 +0,0 @@ - -#include - -/* #undef ENABLE_DOUBLE */ - -#ifdef ENABLE_DOUBLE -# define csf_float double -# define csf_ceil ceil -# define csf_floor floor -# define csf_sin sin -# define csf_log log -# define csf_log10 log10 -# define csf_pow pow -# define csf_sqrt sqrt -# define csf_float_min DBL_MIN -#else -# define csf_float float -# define csf_ceil ceilf -# define csf_floor floorf -# define csf_sin sinf -# define csf_log logf -# define csf_log10 log10f -# define csf_pow powf -# define csf_sqrt sqrtf -# define csf_float_min FLT_MIN -#endif diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index 80689fa0be..f3c1977d2b 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -16,7 +16,7 @@ const float NUM_FLT_LOGE = 0.4342944819; inline void check( bool x, const char *expr, const char *file, int line, const char *err) { if (!x) { - std::cout << "[" << file << ":" << line << "] "; + std::cerr << "[" << file << ":" << line << "] "; LOG(FATAL) << "\"" << expr << "\" check failed. " << err; } } diff --git a/native_client/deepspeech.cc b/native_client/deepspeech.cc index 31418072b6..a4466cb198 100644 --- a/native_client/deepspeech.cc +++ b/native_client/deepspeech.cc @@ -13,6 +13,7 @@ #include "alphabet.h" #include "native_client/ds_version.h" +#include "native_client/ds_graph_version.h" #ifndef USE_TFLITE #include "tensorflow/core/public/session.h" @@ -23,8 +24,6 @@ #include "tensorflow/lite/kernels/register.h" #endif // USE_TFLITE -#include "c_speech_features.h" - #include "ctcdecode/ctc_beam_search_decoder.h" #ifdef __ANDROID__ @@ -48,14 +47,6 @@ constexpr float AUDIO_WIN_STEP = 0.02f; constexpr unsigned int AUDIO_WIN_LEN_SAMPLES = (unsigned int)(AUDIO_WIN_LEN * SAMPLE_RATE); constexpr unsigned int AUDIO_WIN_STEP_SAMPLES = (unsigned int)(AUDIO_WIN_STEP * SAMPLE_RATE); -constexpr unsigned int MFCC_FEATURES = 26; - -constexpr float PREEMPHASIS_COEFF = 0.97f; -constexpr unsigned int N_FFT = 512; -constexpr unsigned int N_FILTERS = 26; -constexpr unsigned int LOWFREQ = 0; -constexpr unsigned int CEP_LIFTER = 22; - constexpr size_t WINDOW_SIZE = AUDIO_WIN_LEN * SAMPLE_RATE; std::array calc_hamming_window() { @@ -108,7 +99,6 @@ using std::vector; struct StreamingState { vector accumulated_logits; vector audio_buffer; - float last_sample; // used for preemphasis vector mfcc_buffer; vector batch_buffer; ModelState* model; @@ -121,7 +111,7 @@ struct StreamingState { void processAudioWindow(const vector& buf); void processMfccWindow(const vector& buf); - void pushMfccBuffer(const float* buf, unsigned int len); + void pushMfccBuffer(const vector& buf); void addZeroMfccWindow(); void processBatch(const vector& buf, unsigned int n_steps); }; @@ -141,8 +131,9 @@ struct ModelState { Scorer* scorer; unsigned int beam_width; unsigned int n_steps; - unsigned int mfcc_feats_per_timestep; unsigned int n_context; + unsigned int n_features; + unsigned int mfcc_feats_per_timestep; #ifdef USE_TFLITE size_t previous_state_size; @@ -152,10 +143,12 @@ struct ModelState { int input_node_idx; int previous_state_c_idx; int previous_state_h_idx; + int input_samples_idx; int logits_idx; int new_state_c_idx; int new_state_h_idx; + int mfccs_idx; #endif ModelState(); @@ -170,7 +163,7 @@ struct ModelState { * * @return String representing the decoded text. */ - char* decode(vector& logits); + char* decode(const vector& logits); /** * @brief Perform decoding of the logits, using basic CTC decoder or @@ -192,7 +185,7 @@ struct ModelState { * @return Metadata struct containing MetadataItem structs for each character. * The user is responsible for freeing Metadata by calling DS_FreeMetadata(). */ - Metadata* decode_metadata(vector& logits); + Metadata* decode_metadata(const vector& logits); /** * @brief Do a single inference step in the acoustic model, with: @@ -204,11 +197,10 @@ struct ModelState { * * @param[out] output_logits Where to store computed logits. */ - void infer(const float* mfcc, unsigned int n_frames, vector& output_logits); -}; + void infer(const float* mfcc, unsigned int n_frames, vector& logits_output); -StreamingState* setupStreamAndFeedAudioContent(ModelState* aCtx, const short* aBuffer, - unsigned int aBufferSize, unsigned int aSampleRate); + void compute_mfcc(const vector& audio_buffer, vector& mfcc_output); +}; ModelState::ModelState() : @@ -225,8 +217,9 @@ ModelState::ModelState() , scorer(nullptr) , beam_width(0) , n_steps(-1) - , mfcc_feats_per_timestep(-1) , n_context(-1) + , n_features(-1) + , mfcc_feats_per_timestep(-1) #ifdef USE_TFLITE , previous_state_size(0) , previous_state_c_(nullptr) @@ -251,6 +244,14 @@ ModelState::~ModelState() delete alphabet; } +template +void +shift_buffer_left(vector& buf, int shift_amount) +{ + std::rotate(buf.begin(), buf.begin() + shift_amount, buf.end()); + buf.resize(buf.size() - shift_amount); +} + void StreamingState::feedAudioContent(const short* buffer, unsigned int buffer_size) @@ -258,10 +259,9 @@ StreamingState::feedAudioContent(const short* buffer, // Consume all the data that was passed in, processing full buffers if needed while (buffer_size > 0) { while (buffer_size > 0 && audio_buffer.size() < AUDIO_WIN_LEN_SAMPLES) { - // Apply preemphasis to input sample and buffer it - float sample = (float)(*buffer) - (PREEMPHASIS_COEFF * last_sample); - audio_buffer.push_back(sample); - last_sample = *buffer; + // Convert i16 sample into f32 + float multiplier = 1.0f / (1 << 15); + audio_buffer.push_back((float)(*buffer) * multiplier); ++buffer; --buffer_size; } @@ -270,8 +270,7 @@ StreamingState::feedAudioContent(const short* buffer, if (audio_buffer.size() == AUDIO_WIN_LEN_SAMPLES) { processAudioWindow(audio_buffer); // Shift data by one step - std::rotate(audio_buffer.begin(), audio_buffer.begin() + AUDIO_WIN_STEP_SAMPLES, audio_buffer.end()); - audio_buffer.resize(audio_buffer.size() - AUDIO_WIN_STEP_SAMPLES); + shift_buffer_left(audio_buffer, AUDIO_WIN_STEP_SAMPLES); } // Repeat until buffer empty @@ -288,7 +287,6 @@ char* StreamingState::finishStream() { finalizeStream(); - return model->decode(accumulated_logits); } @@ -296,7 +294,6 @@ Metadata* StreamingState::finishStreamWithMetadata() { finalizeStream(); - return model->decode_metadata(accumulated_logits); } @@ -304,15 +301,10 @@ void StreamingState::processAudioWindow(const vector& buf) { // Compute MFCC features - float* mfcc; - int n_frames = csf_mfcc(buf.data(), buf.size(), SAMPLE_RATE, - AUDIO_WIN_LEN, AUDIO_WIN_STEP, MFCC_FEATURES, N_FILTERS, N_FFT, - LOWFREQ, SAMPLE_RATE/2, 0.f, CEP_LIFTER, 1, hamming_window.data(), - &mfcc); - assert(n_frames == 1); - - pushMfccBuffer(mfcc, n_frames * MFCC_FEATURES); - free(mfcc); + vector mfcc; + mfcc.reserve(model->n_features); + model->compute_mfcc(buf, mfcc); + pushMfccBuffer(mfcc); } void @@ -335,25 +327,35 @@ StreamingState::finalizeStream() void StreamingState::addZeroMfccWindow() { - static const float zero_buffer[MFCC_FEATURES] = {0.f}; - pushMfccBuffer(zero_buffer, MFCC_FEATURES); + vector zero_buffer(model->n_features, 0.f); + pushMfccBuffer(zero_buffer); +} + +template +InputIt +copy_up_to_n(InputIt from_begin, InputIt from_end, OutputIt to_begin, int max_elems) +{ + int next_copy_amount = std::min(std::distance(from_begin, from_end), max_elems); + std::copy_n(from_begin, next_copy_amount, to_begin); + return from_begin + next_copy_amount; } void -StreamingState::pushMfccBuffer(const float* buf, unsigned int len) +StreamingState::pushMfccBuffer(const vector& buf) { - while (len > 0) { - unsigned int next_copy_amount = std::min(len, (unsigned int)(model->mfcc_feats_per_timestep - mfcc_buffer.size())); - mfcc_buffer.insert(mfcc_buffer.end(), buf, buf + next_copy_amount); - buf += next_copy_amount; - len -= next_copy_amount; + auto start = buf.begin(); + auto end = buf.end(); + while (start != end) { + // Copy from input buffer to mfcc_buffer, stopping if we have a full context window + start = copy_up_to_n(start, end, std::back_inserter(mfcc_buffer), + model->mfcc_feats_per_timestep - mfcc_buffer.size()); assert(mfcc_buffer.size() <= model->mfcc_feats_per_timestep); + // If we have a full context window if (mfcc_buffer.size() == model->mfcc_feats_per_timestep) { processMfccWindow(mfcc_buffer); // Shift data by one step of one mfcc feature vector - std::rotate(mfcc_buffer.begin(), mfcc_buffer.begin() + MFCC_FEATURES, mfcc_buffer.end()); - mfcc_buffer.resize(mfcc_buffer.size() - MFCC_FEATURES); + shift_buffer_left(mfcc_buffer, model->n_features); } } } @@ -364,11 +366,12 @@ StreamingState::processMfccWindow(const vector& buf) auto start = buf.begin(); auto end = buf.end(); while (start != end) { - unsigned int next_copy_amount = std::min(std::distance(start, end), (unsigned int)(model->n_steps * model->mfcc_feats_per_timestep - batch_buffer.size())); - batch_buffer.insert(batch_buffer.end(), start, start + next_copy_amount); - start += next_copy_amount; + // Copy from input buffer to batch_buffer, stopping if we have a full batch + start = copy_up_to_n(start, end, std::back_inserter(batch_buffer), + model->n_steps * model->mfcc_feats_per_timestep - batch_buffer.size()); assert(batch_buffer.size() <= model->n_steps * model->mfcc_feats_per_timestep); + // If we have a full batch if (batch_buffer.size() == model->n_steps * model->mfcc_feats_per_timestep) { processBatch(batch_buffer, model->n_steps); batch_buffer.resize(0); @@ -388,7 +391,7 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector& logi const size_t num_classes = alphabet->GetSize() + 1; // +1 for blank #ifndef USE_TFLITE - Tensor input(DT_FLOAT, TensorShape({BATCH_SIZE, n_steps, 2*n_context+1, MFCC_FEATURES})); + Tensor input(DT_FLOAT, TensorShape({BATCH_SIZE, n_steps, 2*n_context+1, n_features})); auto input_mapped = input.flat(); int i; @@ -396,7 +399,7 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector& logi input_mapped(i) = aMfcc[i]; } for (; i < n_steps*mfcc_feats_per_timestep; ++i) { - input_mapped(i) = 0; + input_mapped(i) = 0.; } Tensor input_lengths(DT_INT32, TensorShape({1})); @@ -454,11 +457,69 @@ ModelState::infer(const float* aMfcc, unsigned int n_frames, vector& logi #endif // USE_TFLITE } +void +ModelState::compute_mfcc(const vector& samples, vector& mfcc_output) +{ +#ifndef USE_TFLITE + Tensor input(DT_FLOAT, TensorShape({AUDIO_WIN_LEN_SAMPLES})); + auto input_mapped = input.flat(); + int i; + for (i = 0; i < samples.size(); ++i) { + input_mapped(i) = samples[i]; + } + for (; i < AUDIO_WIN_LEN_SAMPLES; ++i) { + input_mapped(i) = 0.f; + } + + vector outputs; + Status status = session->Run({{"input_samples", input}}, {"mfccs"}, {}, &outputs); + + if (!status.ok()) { + std::cerr << "Error running session: " << status << "\n"; + return; + } + + // The feature computation graph is hardcoded to one audio length for now + const int n_windows = 1; + assert(outputs[0].shape().num_elemements() / n_features == n_windows); + + auto mfcc_mapped = outputs[0].flat(); + for (int i = 0; i < n_windows * n_features; ++i) { + mfcc_output.push_back(mfcc_mapped(i)); + } +#else + // Feeding input_node + float* input_samples = interpreter->typed_tensor(input_samples_idx); + for (int i = 0; i < samples.size(); ++i) { + input_samples[i] = samples[i]; + } + + TfLiteStatus status = interpreter->Invoke(); + if (status != kTfLiteOk) { + std::cerr << "Error running session: " << status << "\n"; + return; + } + + // The feature computation graph is hardcoded to one audio length for now + int n_windows = 1; + TfLiteIntArray* out_dims = interpreter->tensor(mfccs_idx)->dims; + int num_elements = 1; + for (int i = 0; i < out_dims->size; ++i) { + num_elements *= out_dims->data[i]; + } + assert(num_elements / n_features == n_windows); + + float* outputs = interpreter->typed_tensor(mfccs_idx); + for (int i = 0; i < n_windows * n_features; ++i) { + mfcc_output.push_back(outputs[i]); + } +#endif +} + char* -ModelState::decode(vector& logits) +ModelState::decode(const vector& logits) { vector out = ModelState::decode_raw(logits); - return strdup(alphabet->LabelsToString(out[0].tokens).c_str()); } @@ -481,7 +542,8 @@ ModelState::decode_raw(const vector& logits) return out; } -Metadata* ModelState::decode_metadata(vector& logits) +Metadata* +ModelState::decode_metadata(const vector& logits) { vector out = decode_raw(logits); @@ -505,7 +567,8 @@ Metadata* ModelState::decode_metadata(vector& logits) } #ifdef USE_TFLITE -int tflite_get_tensor_by_name(const ModelState* ctx, const vector& list, const char* name) +int +tflite_get_tensor_by_name(const ModelState* ctx, const vector& list, const char* name) { int rv = -1; @@ -520,12 +583,14 @@ int tflite_get_tensor_by_name(const ModelState* ctx, const vector& list, co return rv; } -int tflite_get_input_tensor_by_name(const ModelState* ctx, const char* name) +int +tflite_get_input_tensor_by_name(const ModelState* ctx, const char* name) { return ctx->interpreter->inputs()[tflite_get_tensor_by_name(ctx, ctx->interpreter->inputs(), name)]; } -int tflite_get_output_tensor_by_name(const ModelState* ctx, const char* name) +int +tflite_get_output_tensor_by_name(const ModelState* ctx, const char* name) { return ctx->interpreter->outputs()[tflite_get_tensor_by_name(ctx, ctx->interpreter->outputs(), name)]; } @@ -601,12 +666,23 @@ DS_CreateModel(const char* aModelPath, return DS_ERR_FAIL_CREATE_SESS; } + int graph_version = model->graph_def.version(); + if (graph_version < DS_GRAPH_VERSION) { + std::cerr << "Specified model file version (" << graph_version << ") is " + << "incompatible with minimum version supported by this client (" + << DS_GRAPH_VERSION << "). See " + << "https://github.com/mozilla/DeepSpeech/#model-compatibility " + << "for more information" << std::endl; + return DS_ERR_MODEL_INCOMPATIBLE; + } + for (int i = 0; i < model->graph_def.node_size(); ++i) { NodeDef node = model->graph_def.node(i); if (node.name() == "input_node") { const auto& shape = node.attr().at("shape").shape(); model->n_steps = shape.dim(1).size(); model->n_context = (shape.dim(2).size()-1)/2; + model->n_features = shape.dim(3).size(); model->mfcc_feats_per_timestep = shape.dim(2).size() * shape.dim(3).size(); } else if (node.name() == "logits_shape") { Tensor logits_shape = Tensor(DT_INT32, TensorShape({3})); @@ -627,12 +703,10 @@ DS_CreateModel(const char* aModelPath, } } - if (model->n_context == -1) { - std::cerr << "Error: Could not infer context window size from model file. " - << "Make sure input_node is a 3D tensor with the last dimension " - << "of size MFCC_FEATURES * ((2 * context window) + 1). If you " - << "changed the number of features in the input, adjust the " - << "MFCC_FEATURES constant in " __FILE__ + if (model->n_context == -1 || model->n_features == -1) { + std::cerr << "Error: Could not infer input shape from model file. " + << "Make sure input_node is a 4D tensor with shape " + << "[batch_size=1, time, window_size, n_features]." << std::endl; return DS_ERR_INVALID_SHAPE; } @@ -640,8 +714,6 @@ DS_CreateModel(const char* aModelPath, *retval = model.release(); return DS_ERR_OK; #else // USE_TFLITE - TfLiteStatus status; - model->fbmodel = tflite::FlatBufferModel::BuildFromFile(aModelPath); if (!model->fbmodel) { std::cerr << "Error at reading model file " << aModelPath << std::endl; @@ -663,14 +735,17 @@ DS_CreateModel(const char* aModelPath, model->input_node_idx = tflite_get_input_tensor_by_name(model.get(), "input_node"); model->previous_state_c_idx = tflite_get_input_tensor_by_name(model.get(), "previous_state_c"); model->previous_state_h_idx = tflite_get_input_tensor_by_name(model.get(), "previous_state_h"); + model->input_samples_idx = tflite_get_input_tensor_by_name(model.get(), "input_samples"); model->logits_idx = tflite_get_output_tensor_by_name(model.get(), "logits"); model->new_state_c_idx = tflite_get_output_tensor_by_name(model.get(), "new_state_c"); model->new_state_h_idx = tflite_get_output_tensor_by_name(model.get(), "new_state_h"); + model->mfccs_idx = tflite_get_output_tensor_by_name(model.get(), "mfccs"); TfLiteIntArray* dims_input_node = model->interpreter->tensor(model->input_node_idx)->dims; model->n_steps = dims_input_node->data[1]; model->n_context = (dims_input_node->data[2] - 1 ) / 2; + model->n_features = dims_input_node->data[3]; model->mfcc_feats_per_timestep = dims_input_node->data[2] * dims_input_node->data[3]; TfLiteIntArray* dims_logits = model->interpreter->tensor(model->logits_idx)->dims; @@ -727,43 +802,6 @@ DS_EnableDecoderWithLM(ModelState* aCtx, } } -char* -DS_SpeechToText(ModelState* aCtx, - const short* aBuffer, - unsigned int aBufferSize, - unsigned int aSampleRate) -{ - StreamingState* ctx = setupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); - return DS_FinishStream(ctx); -} - -Metadata* -DS_SpeechToTextWithMetadata(ModelState* aCtx, - const short* aBuffer, - unsigned int aBufferSize, - unsigned int aSampleRate) -{ - StreamingState* ctx = setupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); - return DS_FinishStreamWithMetadata(ctx); -} - -StreamingState* -setupStreamAndFeedAudioContent(ModelState* aCtx, - const short* aBuffer, - unsigned int aBufferSize, - unsigned int aSampleRate) -{ - StreamingState* ctx; - int status = DS_SetupStream(aCtx, 0, aSampleRate, &ctx); - if (status != DS_ERR_OK) { - return nullptr; - } - - DS_FeedAudioContent(ctx, aBuffer, aBufferSize); - - return ctx; -} - int DS_SetupStream(ModelState* aCtx, unsigned int aPreAllocFrames, @@ -796,9 +834,8 @@ DS_SetupStream(ModelState* aCtx, ctx->accumulated_logits.reserve(aPreAllocFrames * BATCH_SIZE * num_classes); ctx->audio_buffer.reserve(AUDIO_WIN_LEN_SAMPLES); - ctx->last_sample = 0; ctx->mfcc_buffer.reserve(aCtx->mfcc_feats_per_timestep); - ctx->mfcc_buffer.resize(MFCC_FEATURES*aCtx->n_context, 0.f); + ctx->mfcc_buffer.resize(aCtx->n_features*aCtx->n_context, 0.f); ctx->batch_buffer.reserve(aCtx->n_steps * aCtx->mfcc_feats_per_timestep); ctx->model = aCtx; @@ -837,83 +874,50 @@ DS_FinishStreamWithMetadata(StreamingState* aSctx) return metadata; } -void -DS_DiscardStream(StreamingState* aSctx) +StreamingState* +SetupStreamAndFeedAudioContent(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate) { - delete aSctx; + StreamingState* ctx; + int status = DS_SetupStream(aCtx, 0, aSampleRate, &ctx); + if (status != DS_ERR_OK) { + return nullptr; + } + DS_FeedAudioContent(ctx, aBuffer, aBufferSize); + return ctx; } -void -DS_AudioToInputVector(const short* aBuffer, - unsigned int aBufferSize, - unsigned int aSampleRate, - unsigned int aNCep, - unsigned int aNContext, - float** aMfcc, - int* aNFrames, - int* aFrameLen) -{ - const int contextSize = aNCep * aNContext; - const int frameSize = aNCep + (2 * aNCep * aNContext); - - // Compute MFCC features - float* mfcc; - int n_frames = csf_mfcc(aBuffer, aBufferSize, aSampleRate, - AUDIO_WIN_LEN, AUDIO_WIN_STEP, aNCep, N_FILTERS, N_FFT, - LOWFREQ, aSampleRate/2, PREEMPHASIS_COEFF, CEP_LIFTER, - 1, NULL, &mfcc); - - // Take every other frame (BiRNN stride of 2) and add past/future context - int ds_input_length = (n_frames + 1) / 2; - // TODO: Use MFCC of silence instead of zero - float* ds_input = (float*)calloc(ds_input_length * frameSize, sizeof(float)); - for (int i = 0, idx = 0, mfcc_idx = 0; i < ds_input_length; - i++, idx += frameSize, mfcc_idx += aNCep * 2) { - // Past context - for (int j = aNContext; j > 0; j--) { - int frame_index = (i - j) * 2; - if (frame_index < 0) { continue; } - int mfcc_base = frame_index * aNCep; - int base = (aNContext - j) * aNCep; - for (int k = 0; k < aNCep; k++) { - ds_input[idx + base + k] = mfcc[mfcc_base + k]; - } - } - - // Present context - for (int j = 0; j < aNCep; j++) { - ds_input[idx + j + contextSize] = mfcc[mfcc_idx + j]; - } - - // Future context - for (int j = 1; j <= aNContext; j++) { - int frame_index = (i + j) * 2; - if (frame_index >= n_frames) { break; } - int mfcc_base = frame_index * aNCep; - int base = contextSize + aNCep + ((j - 1) * aNCep); - for (int k = 0; k < aNCep; k++) { - ds_input[idx + base + k] = mfcc[mfcc_base + k]; - } - } - } +char* +DS_SpeechToText(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate) +{ + StreamingState* ctx = SetupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); + return DS_FinishStream(ctx); +} - // Free mfcc array - free(mfcc); +Metadata* +DS_SpeechToTextWithMetadata(ModelState* aCtx, + const short* aBuffer, + unsigned int aBufferSize, + unsigned int aSampleRate) +{ + StreamingState* ctx = SetupStreamAndFeedAudioContent(aCtx, aBuffer, aBufferSize, aSampleRate); + return DS_FinishStreamWithMetadata(ctx); +} - if (aMfcc) { - *aMfcc = ds_input; - } - if (aNFrames) { - *aNFrames = ds_input_length; - } - if (aFrameLen) { - *aFrameLen = frameSize; - } +void +DS_DiscardStream(StreamingState* aSctx) +{ + delete aSctx; } -void -DS_FreeMetadata(Metadata* m) -{ +void +DS_FreeMetadata(Metadata* m) +{ if (m) { delete(m->items); delete(m); diff --git a/native_client/deepspeech.h b/native_client/deepspeech.h index acb01eccc7..eb8b230fbe 100644 --- a/native_client/deepspeech.h +++ b/native_client/deepspeech.h @@ -40,6 +40,7 @@ enum DeepSpeech_Error_Codes DS_ERR_INVALID_ALPHABET = 0x2000, DS_ERR_INVALID_SHAPE = 0x2001, DS_ERR_INVALID_LM = 0x2002, + DS_ERR_MODEL_INCOMPATIBLE = 0x2003, // Runtime failures DS_ERR_FAIL_INIT_MMAP = 0x3000, @@ -228,41 +229,9 @@ Metadata* DS_FinishStreamWithMetadata(StreamingState* aSctx); DEEPSPEECH_EXPORT void DS_DiscardStream(StreamingState* aSctx); -/** - * @brief Given audio, return a vector suitable for input to a DeepSpeech - * model trained with the given parameters. - * - * Extracts MFCC features from a given audio signal and adds the appropriate - * amount of context to run inference on a DeepSpeech model trained with - * the given parameters. - * - * @param aBuffer A 16-bit, mono raw audio signal at the appropriate sample - * rate. - * @param aBufferSize The sample-length of the audio signal. - * @param aSampleRate The sample-rate of the audio signal. - * @param aNCep The number of cepstrum. - * @param aNContext The size of the context window. - * @param[out] aMfcc An array containing features, of shape - * (@p aNFrames, ncep * ncontext). The user is responsible - * for freeing the array. - * @param[out] aNFrames (optional) The number of frames in @p aMfcc. - * @param[out] aFrameLen (optional) The length of each frame - * (ncep * ncontext) in @p aMfcc. - */ -DEEPSPEECH_EXPORT -void DS_AudioToInputVector(const short* aBuffer, - unsigned int aBufferSize, - unsigned int aSampleRate, - unsigned int aNCep, - unsigned int aNContext, - float** aMfcc, - int* aNFrames = NULL, - int* aFrameLen = NULL); - /** * @brief Free memory allocated for metadata information. */ - DEEPSPEECH_EXPORT void DS_FreeMetadata(Metadata* m); diff --git a/native_client/ds_graph_version.sh b/native_client/ds_graph_version.sh new file mode 100755 index 0000000000..391b28ddae --- /dev/null +++ b/native_client/ds_graph_version.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ `uname` = "Darwin" ]; then + export PATH="/Users/build-user/TaskCluster/Workdir/tasks/tc-workdir/homebrew/opt/coreutils/libexec/gnubin:${PATH}" +fi + +DS_DIR="$(realpath "$(dirname "$(realpath "$0")")/../")" +if [ ! -d "${DS_DIR}" ]; then + exit 1 +fi; + +DS_GRAPH_VERSION=$(cat "${DS_DIR}/GRAPH_VERSION") +if [ $? -ne 0 ]; then + exit 1 +fi + +cat < array = Array::New(Isolate::GetCurrent(), *$2); - for (unsigned int i = 0, idx = 0; i < *$2; i++) { - Handle buffer = - ArrayBuffer::New(Isolate::GetCurrent(), *$1, *$3 * sizeof(float)); - memcpy(buffer->GetContents().Data(), - (*$1) + (idx += *$3), *$3 * sizeof(float)); - Handle inner = Float32Array::New(buffer, 0, *$3); - array->Set(i, inner); - } - free(*$1); - $result = array; -} - -%apply (float** ARGOUTVIEWM_ARRAY2, unsigned int* DIM1, unsigned int* DIM2) {(float** aMfcc, unsigned int* aNFrames, unsigned int* aFrameLen)}; - // make sure the string returned by SpeechToText is freed %typemap(newfree) char* "free($1);"; %newobject DS_SpeechToText; diff --git a/native_client/javascript/index.js b/native_client/javascript/index.js index 944aafa923..fed5e57454 100644 --- a/native_client/javascript/index.js +++ b/native_client/javascript/index.js @@ -66,6 +66,5 @@ Model.prototype.finishStream = function() { module.exports = { Model: Model, - audioToInputVector: binding.AudioToInputVector, printVersions: binding.PrintVersions }; diff --git a/native_client/kiss_fft130/CHANGELOG b/native_client/kiss_fft130/CHANGELOG deleted file mode 100644 index 2dd3603755..0000000000 --- a/native_client/kiss_fft130/CHANGELOG +++ /dev/null @@ -1,123 +0,0 @@ -1.3.0 2012-07-18 - removed non-standard malloc.h from kiss_fft.h - - moved -lm to end of link line - - checked various return values - - converted python Numeric code to NumPy - - fixed test of int32_t on 64 bit OS - - added padding in a couple of places to allow SIMD alignment of structs - -1.2.9 2010-05-27 - threadsafe ( including OpenMP ) - - first edition of kissfft.hh the C++ template fft engine - -1.2.8 - Changed memory.h to string.h -- apparently more standard - - Added openmp extensions. This can have fairly linear speedups for larger FFT sizes. - -1.2.7 - Shrank the real-fft memory footprint. Thanks to Galen Seitz. - -1.2.6 (Nov 14, 2006) The "thanks to GenArts" release. - Added multi-dimensional real-optimized FFT, see tools/kiss_fftndr - Thanks go to GenArts, Inc. for sponsoring the development. - -1.2.5 (June 27, 2006) The "release for no good reason" release. - Changed some harmless code to make some compilers' warnings go away. - Added some more digits to pi -- why not. - Added kiss_fft_next_fast_size() function to help people decide how much to pad. - Changed multidimensional test from 8 dimensions to only 3 to avoid testing - problems with fixed point (sorry Buckaroo Banzai). - -1.2.4 (Oct 27, 2005) The "oops, inverse fixed point real fft was borked" release. - Fixed scaling bug for inverse fixed point real fft -- also fixed test code that should've been failing. - Thanks to Jean-Marc Valin for bug report. - - Use sys/types.h for more portable types than short,int,long => int16_t,int32_t,int64_t - If your system does not have these, you may need to define them -- but at least it breaks in a - loud and easily fixable way -- unlike silently using the wrong size type. - - Hopefully tools/psdpng.c is fixed -- thanks to Steve Kellog for pointing out the weirdness. - -1.2.3 (June 25, 2005) The "you want to use WHAT as a sample" release. - Added ability to use 32 bit fixed point samples -- requires a 64 bit intermediate result, a la 'long long' - - Added ability to do 4 FFTs in parallel by using SSE SIMD instructions. This is accomplished by - using the __m128 (vector of 4 floats) as kiss_fft_scalar. Define USE_SIMD to use this. - - I know, I know ... this is drifting a bit from the "kiss" principle, but the speed advantages - make it worth it for some. Also recent gcc makes it SOO easy to use vectors of 4 floats like a POD type. - -1.2.2 (May 6, 2005) The Matthew release - Replaced fixed point division with multiply&shift. Thanks to Jean-Marc Valin for - discussions regarding. Considerable speedup for fixed-point. - - Corrected overflow protection in real fft routines when using fixed point. - Finder's Credit goes to Robert Oschler of robodance for pointing me at the bug. - This also led to the CHECK_OVERFLOW_OP macro. - -1.2.1 (April 4, 2004) - compiles cleanly with just about every -W warning flag under the sun - - reorganized kiss_fft_state so it could be read-only/const. This may be useful for embedded systems - that are willing to predeclare twiddle factors, factorization. - - Fixed C_MUL,S_MUL on 16-bit platforms. - - tmpbuf will only be allocated if input & output buffers are same - scratchbuf will only be allocated for ffts that are not multiples of 2,3,5 - - NOTE: The tmpbuf,scratchbuf changes may require synchronization code for multi-threaded apps. - - -1.2 (Feb 23, 2004) - interface change -- cfg object is forward declaration of struct instead of void* - This maintains type saftey and lets the compiler warn/error about stupid mistakes. - (prompted by suggestion from Erik de Castro Lopo) - - small speed improvements - - added psdpng.c -- sample utility that will create png spectrum "waterfalls" from an input file - ( not terribly useful yet) - -1.1.1 (Feb 1, 2004 ) - minor bug fix -- only affects odd rank, in-place, multi-dimensional FFTs - -1.1 : (Jan 30,2004) - split sample_code/ into test/ and tools/ - - Removed 2-D fft and added N-D fft (arbitrary) - - modified fftutil.c to allow multi-d FFTs - - Modified core fft routine to allow an input stride via kiss_fft_stride() - (eased support of multi-D ffts) - - Added fast convolution filtering (FIR filtering using overlap-scrap method, with tail scrap) - - Add kfc.[ch]: the KISS FFT Cache. It takes care of allocs for you ( suggested by Oscar Lesta ). - -1.0.1 (Dec 15, 2003) - fixed bug that occurred when nfft==1. Thanks to Steven Johnson. - -1.0 : (Dec 14, 2003) - changed kiss_fft function from using a single buffer, to two buffers. - If the same buffer pointer is supplied for both in and out, kiss will - manage the buffer copies. - - added kiss_fft2d and kiss_fftr as separate source files (declarations in kiss_fft.h ) - -0.4 :(Nov 4,2003) optimized for radix 2,3,4,5 - -0.3 :(Oct 28, 2003) woops, version 2 didn't actually factor out any radices other than 2. - Thanks to Steven Johnson for finding this one. - -0.2 :(Oct 27, 2003) added mixed radix, only radix 2,4 optimized versions - -0.1 :(May 19 2003) initial release, radix 2 only diff --git a/native_client/kiss_fft130/COPYING b/native_client/kiss_fft130/COPYING deleted file mode 100644 index 2fc6685a6d..0000000000 --- a/native_client/kiss_fft130/COPYING +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) 2003-2010 Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/native_client/kiss_fft130/Makefile b/native_client/kiss_fft130/Makefile deleted file mode 100644 index 96f43d3f3e..0000000000 --- a/native_client/kiss_fft130/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -KFVER=130 - -doc: - @echo "Start by reading the README file. If you want to build and test lots of stuff, do a 'make testall'" - @echo "but be aware that 'make testall' has dependencies that the basic kissfft software does not." - @echo "It is generally unneeded to run these tests yourself, unless you plan on changing the inner workings" - @echo "of kissfft and would like to make use of its regression tests." - -testall: - # The simd and int32_t types may or may not work on your machine - make -C test DATATYPE=simd CFLAGADD="$(CFLAGADD)" test - make -C test DATATYPE=int32_t CFLAGADD="$(CFLAGADD)" test - make -C test DATATYPE=int16_t CFLAGADD="$(CFLAGADD)" test - make -C test DATATYPE=float CFLAGADD="$(CFLAGADD)" test - make -C test DATATYPE=double CFLAGADD="$(CFLAGADD)" test - echo "all tests passed" - -tarball: clean - hg archive -r v$(KFVER) -t tgz kiss_fft$(KFVER).tar.gz - hg archive -r v$(KFVER) -t zip kiss_fft$(KFVER).zip - -clean: - cd test && make clean - cd tools && make clean - rm -f kiss_fft*.tar.gz *~ *.pyc kiss_fft*.zip - -asm: kiss_fft.s - -kiss_fft.s: kiss_fft.c kiss_fft.h _kiss_fft_guts.h - [ -e kiss_fft.s ] && mv kiss_fft.s kiss_fft.s~ || true - gcc -S kiss_fft.c -O3 -mtune=native -ffast-math -fomit-frame-pointer -unroll-loops -dA -fverbose-asm - gcc -o kiss_fft_short.s -S kiss_fft.c -O3 -mtune=native -ffast-math -fomit-frame-pointer -dA -fverbose-asm -DFIXED_POINT - [ -e kiss_fft.s~ ] && diff kiss_fft.s~ kiss_fft.s || true diff --git a/native_client/kiss_fft130/README b/native_client/kiss_fft130/README deleted file mode 100644 index 03b2e7a9c1..0000000000 --- a/native_client/kiss_fft130/README +++ /dev/null @@ -1,134 +0,0 @@ -KISS FFT - A mixed-radix Fast Fourier Transform based up on the principle, -"Keep It Simple, Stupid." - - There are many great fft libraries already around. Kiss FFT is not trying -to be better than any of them. It only attempts to be a reasonably efficient, -moderately useful FFT that can use fixed or floating data types and can be -incorporated into someone's C program in a few minutes with trivial licensing. - -USAGE: - - The basic usage for 1-d complex FFT is: - - #include "kiss_fft.h" - - kiss_fft_cfg cfg = kiss_fft_alloc( nfft ,is_inverse_fft ,0,0 ); - - while ... - - ... // put kth sample in cx_in[k].r and cx_in[k].i - - kiss_fft( cfg , cx_in , cx_out ); - - ... // transformed. DC is in cx_out[0].r and cx_out[0].i - - free(cfg); - - Note: frequency-domain data is stored from dc up to 2pi. - so cx_out[0] is the dc bin of the FFT - and cx_out[nfft/2] is the Nyquist bin (if exists) - - Declarations are in "kiss_fft.h", along with a brief description of the -functions you'll need to use. - -Code definitions for 1d complex FFTs are in kiss_fft.c. - -You can do other cool stuff with the extras you'll find in tools/ - - * multi-dimensional FFTs - * real-optimized FFTs (returns the positive half-spectrum: (nfft/2+1) complex frequency bins) - * fast convolution FIR filtering (not available for fixed point) - * spectrum image creation - -The core fft and most tools/ code can be compiled to use float, double, - Q15 short or Q31 samples. The default is float. - - -BACKGROUND: - - I started coding this because I couldn't find a fixed point FFT that didn't -use assembly code. I started with floating point numbers so I could get the -theory straight before working on fixed point issues. In the end, I had a -little bit of code that could be recompiled easily to do ffts with short, float -or double (other types should be easy too). - - Once I got my FFT working, I was curious about the speed compared to -a well respected and highly optimized fft library. I don't want to criticize -this great library, so let's call it FFT_BRANDX. -During this process, I learned: - - 1. FFT_BRANDX has more than 100K lines of code. The core of kiss_fft is about 500 lines (cpx 1-d). - 2. It took me an embarrassingly long time to get FFT_BRANDX working. - 3. A simple program using FFT_BRANDX is 522KB. A similar program using kiss_fft is 18KB (without optimizing for size). - 4. FFT_BRANDX is roughly twice as fast as KISS FFT in default mode. - - It is wonderful that free, highly optimized libraries like FFT_BRANDX exist. -But such libraries carry a huge burden of complexity necessary to extract every -last bit of performance. - - Sometimes simpler is better, even if it's not better. - -FREQUENTLY ASKED QUESTIONS: - Q: Can I use kissfft in a project with a ___ license? - A: Yes. See LICENSE below. - - Q: Why don't I get the output I expect? - A: The two most common causes of this are - 1) scaling : is there a constant multiplier between what you got and what you want? - 2) mixed build environment -- all code must be compiled with same preprocessor - definitions for FIXED_POINT and kiss_fft_scalar - - Q: Will you write/debug my code for me? - A: Probably not unless you pay me. I am happy to answer pointed and topical questions, but - I may refer you to a book, a forum, or some other resource. - - -PERFORMANCE: - (on Athlon XP 2100+, with gcc 2.96, float data type) - - Kiss performed 10000 1024-pt cpx ffts in .63 s of cpu time. - For comparison, it took md5sum twice as long to process the same amount of data. - - Transforming 5 minutes of CD quality audio takes less than a second (nfft=1024). - -DO NOT: - ... use Kiss if you need the Fastest Fourier Transform in the World - ... ask me to add features that will bloat the code - -UNDER THE HOOD: - - Kiss FFT uses a time decimation, mixed-radix, out-of-place FFT. If you give it an input buffer - and output buffer that are the same, a temporary buffer will be created to hold the data. - - No static data is used. The core routines of kiss_fft are thread-safe (but not all of the tools directory). - - No scaling is done for the floating point version (for speed). - Scaling is done both ways for the fixed-point version (for overflow prevention). - - Optimized butterflies are used for factors 2,3,4, and 5. - - The real (i.e. not complex) optimization code only works for even length ffts. It does two half-length - FFTs in parallel (packed into real&imag), and then combines them via twiddling. The result is - nfft/2+1 complex frequency bins from DC to Nyquist. If you don't know what this means, search the web. - - The fast convolution filtering uses the overlap-scrap method, slightly - modified to put the scrap at the tail. - -LICENSE: - Revised BSD License, see COPYING for verbiage. - Basically, "free to use&change, give credit where due, no guarantees" - Note this license is compatible with GPL at one end of the spectrum and closed, commercial software at - the other end. See http://www.fsf.org/licensing/licenses - - A commercial license is available which removes the requirement for attribution. Contact me for details. - - -TODO: - *) Add real optimization for odd length FFTs - *) Document/revisit the input/output fft scaling - *) Make doc describing the overlap (tail) scrap fast convolution filtering in kiss_fastfir.c - *) Test all the ./tools/ code with fixed point (kiss_fastfir.c doesn't work, maybe others) - -AUTHOR: - Mark Borgerding - Mark@Borgerding.net diff --git a/native_client/kiss_fft130/README.simd b/native_client/kiss_fft130/README.simd deleted file mode 100644 index b0fdac5506..0000000000 --- a/native_client/kiss_fft130/README.simd +++ /dev/null @@ -1,78 +0,0 @@ -If you are reading this, it means you think you may be interested in using the SIMD extensions in kissfft -to do 4 *separate* FFTs at once. - -Beware! Beyond here there be dragons! - -This API is not easy to use, is not well documented, and breaks the KISS principle. - - -Still reading? Okay, you may get rewarded for your patience with a considerable speedup -(2-3x) on intel x86 machines with SSE if you are willing to jump through some hoops. - -The basic idea is to use the packed 4 float __m128 data type as a scalar element. -This means that the format is pretty convoluted. It performs 4 FFTs per fft call on signals A,B,C,D. - -For complex data, the data is interlaced as follows: -rA0,rB0,rC0,rD0, iA0,iB0,iC0,iD0, rA1,rB1,rC1,rD1, iA1,iB1,iC1,iD1 ... -where "rA0" is the real part of the zeroth sample for signal A - -Real-only data is laid out: -rA0,rB0,rC0,rD0, rA1,rB1,rC1,rD1, ... - -Compile with gcc flags something like --O3 -mpreferred-stack-boundary=4 -DUSE_SIMD=1 -msse - -Be aware of SIMD alignment. This is the most likely cause of segfaults. -The code within kissfft uses scratch variables on the stack. -With SIMD, these must have addresses on 16 byte boundaries. -Search on "SIMD alignment" for more info. - - - -Robin at Divide Concept was kind enough to share his code for formatting to/from the SIMD kissfft. -I have not run it -- use it at your own risk. It appears to do 4xN and Nx4 transpositions -(out of place). - -void SSETools::pack128(float* target, float* source, unsigned long size128) -{ - __m128* pDest = (__m128*)target; - __m128* pDestEnd = pDest+size128; - float* source0=source; - float* source1=source0+size128; - float* source2=source1+size128; - float* source3=source2+size128; - - while(pDest - -#define MAXFACTORS 32 -/* e.g. an fft of length 128 has 4 factors - as far as kissfft is concerned - 4*4*4*2 - */ - -struct kiss_fft_state{ - int nfft; - int inverse; - int factors[2*MAXFACTORS]; - kiss_fft_cpx twiddles[1]; -}; - -/* - Explanation of macros dealing with complex math: - - C_MUL(m,a,b) : m = a*b - C_FIXDIV( c , div ) : if a fixed point impl., c /= div. noop otherwise - C_SUB( res, a,b) : res = a - b - C_SUBFROM( res , a) : res -= a - C_ADDTO( res , a) : res += a - * */ -#ifdef FIXED_POINT -#if (FIXED_POINT==32) -# define FRACBITS 31 -# define SAMPPROD int64_t -#define SAMP_MAX 2147483647 -#else -# define FRACBITS 15 -# define SAMPPROD int32_t -#define SAMP_MAX 32767 -#endif - -#define SAMP_MIN -SAMP_MAX - -#if defined(CHECK_OVERFLOW) -# define CHECK_OVERFLOW_OP(a,op,b) \ - if ( (SAMPPROD)(a) op (SAMPPROD)(b) > SAMP_MAX || (SAMPPROD)(a) op (SAMPPROD)(b) < SAMP_MIN ) { \ - fprintf(stderr,"WARNING:overflow @ " __FILE__ "(%d): (%d " #op" %d) = %ld\n",__LINE__,(a),(b),(SAMPPROD)(a) op (SAMPPROD)(b) ); } -#endif - - -# define smul(a,b) ( (SAMPPROD)(a)*(b) ) -# define sround( x ) (kiss_fft_scalar)( ( (x) + (1<<(FRACBITS-1)) ) >> FRACBITS ) - -# define S_MUL(a,b) sround( smul(a,b) ) - -# define C_MUL(m,a,b) \ - do{ (m).r = sround( smul((a).r,(b).r) - smul((a).i,(b).i) ); \ - (m).i = sround( smul((a).r,(b).i) + smul((a).i,(b).r) ); }while(0) - -# define DIVSCALAR(x,k) \ - (x) = sround( smul( x, SAMP_MAX/k ) ) - -# define C_FIXDIV(c,div) \ - do { DIVSCALAR( (c).r , div); \ - DIVSCALAR( (c).i , div); }while (0) - -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r = sround( smul( (c).r , s ) ) ;\ - (c).i = sround( smul( (c).i , s ) ) ; }while(0) - -#else /* not FIXED_POINT*/ - -# define S_MUL(a,b) ( (a)*(b) ) -#define C_MUL(m,a,b) \ - do{ (m).r = (a).r*(b).r - (a).i*(b).i;\ - (m).i = (a).r*(b).i + (a).i*(b).r; }while(0) -# define C_FIXDIV(c,div) /* NOOP */ -# define C_MULBYSCALAR( c, s ) \ - do{ (c).r *= (s);\ - (c).i *= (s); }while(0) -#endif - -#ifndef CHECK_OVERFLOW_OP -# define CHECK_OVERFLOW_OP(a,op,b) /* noop */ -#endif - -#define C_ADD( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,+,(b).r)\ - CHECK_OVERFLOW_OP((a).i,+,(b).i)\ - (res).r=(a).r+(b).r; (res).i=(a).i+(b).i; \ - }while(0) -#define C_SUB( res, a,b)\ - do { \ - CHECK_OVERFLOW_OP((a).r,-,(b).r)\ - CHECK_OVERFLOW_OP((a).i,-,(b).i)\ - (res).r=(a).r-(b).r; (res).i=(a).i-(b).i; \ - }while(0) -#define C_ADDTO( res , a)\ - do { \ - CHECK_OVERFLOW_OP((res).r,+,(a).r)\ - CHECK_OVERFLOW_OP((res).i,+,(a).i)\ - (res).r += (a).r; (res).i += (a).i;\ - }while(0) - -#define C_SUBFROM( res , a)\ - do {\ - CHECK_OVERFLOW_OP((res).r,-,(a).r)\ - CHECK_OVERFLOW_OP((res).i,-,(a).i)\ - (res).r -= (a).r; (res).i -= (a).i; \ - }while(0) - - -#ifdef FIXED_POINT -# define KISS_FFT_COS(phase) floor(.5+SAMP_MAX * cos (phase)) -# define KISS_FFT_SIN(phase) floor(.5+SAMP_MAX * sin (phase)) -# define HALF_OF(x) ((x)>>1) -#elif defined(USE_SIMD) -# define KISS_FFT_COS(phase) _mm_set1_ps( cos(phase) ) -# define KISS_FFT_SIN(phase) _mm_set1_ps( sin(phase) ) -# define HALF_OF(x) ((x)*_mm_set1_ps(.5)) -#else -# define KISS_FFT_COS(phase) (kiss_fft_scalar) cos(phase) -# define KISS_FFT_SIN(phase) (kiss_fft_scalar) sin(phase) -# define HALF_OF(x) ((x)*.5) -#endif - -#define kf_cexp(x,phase) \ - do{ \ - (x)->r = KISS_FFT_COS(phase);\ - (x)->i = KISS_FFT_SIN(phase);\ - }while(0) - - -/* a debugging function */ -#define pcpx(c)\ - fprintf(stderr,"%g + %gi\n",(double)((c)->r),(double)((c)->i) ) - - -#ifdef KISS_FFT_USE_ALLOCA -// define this to allow use of alloca instead of malloc for temporary buffers -// Temporary buffers are used in two case: -// 1. FFT sizes that have "bad" factors. i.e. not 2,3 and 5 -// 2. "in-place" FFTs. Notice the quotes, since kissfft does not really do an in-place transform. -#include -#define KISS_FFT_TMP_ALLOC(nbytes) alloca(nbytes) -#define KISS_FFT_TMP_FREE(ptr) -#else -#define KISS_FFT_TMP_ALLOC(nbytes) KISS_FFT_MALLOC(nbytes) -#define KISS_FFT_TMP_FREE(ptr) KISS_FFT_FREE(ptr) -#endif diff --git a/native_client/kiss_fft130/kiss_fft.c b/native_client/kiss_fft130/kiss_fft.c deleted file mode 100644 index 465d6c97a0..0000000000 --- a/native_client/kiss_fft130/kiss_fft.c +++ /dev/null @@ -1,408 +0,0 @@ -/* -Copyright (c) 2003-2010, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - - -#include "_kiss_fft_guts.h" -/* The guts header contains all the multiplication and addition macros that are defined for - fixed or floating point complex numbers. It also delares the kf_ internal functions. - */ - -static void kf_bfly2( - kiss_fft_cpx * Fout, - const size_t fstride, - const kiss_fft_cfg st, - int m - ) -{ - kiss_fft_cpx * Fout2; - kiss_fft_cpx * tw1 = st->twiddles; - kiss_fft_cpx t; - Fout2 = Fout + m; - do{ - C_FIXDIV(*Fout,2); C_FIXDIV(*Fout2,2); - - C_MUL (t, *Fout2 , *tw1); - tw1 += fstride; - C_SUB( *Fout2 , *Fout , t ); - C_ADDTO( *Fout , t ); - ++Fout2; - ++Fout; - }while (--m); -} - -static void kf_bfly4( - kiss_fft_cpx * Fout, - const size_t fstride, - const kiss_fft_cfg st, - const size_t m - ) -{ - kiss_fft_cpx *tw1,*tw2,*tw3; - kiss_fft_cpx scratch[6]; - size_t k=m; - const size_t m2=2*m; - const size_t m3=3*m; - - - tw3 = tw2 = tw1 = st->twiddles; - - do { - C_FIXDIV(*Fout,4); C_FIXDIV(Fout[m],4); C_FIXDIV(Fout[m2],4); C_FIXDIV(Fout[m3],4); - - C_MUL(scratch[0],Fout[m] , *tw1 ); - C_MUL(scratch[1],Fout[m2] , *tw2 ); - C_MUL(scratch[2],Fout[m3] , *tw3 ); - - C_SUB( scratch[5] , *Fout, scratch[1] ); - C_ADDTO(*Fout, scratch[1]); - C_ADD( scratch[3] , scratch[0] , scratch[2] ); - C_SUB( scratch[4] , scratch[0] , scratch[2] ); - C_SUB( Fout[m2], *Fout, scratch[3] ); - tw1 += fstride; - tw2 += fstride*2; - tw3 += fstride*3; - C_ADDTO( *Fout , scratch[3] ); - - if(st->inverse) { - Fout[m].r = scratch[5].r - scratch[4].i; - Fout[m].i = scratch[5].i + scratch[4].r; - Fout[m3].r = scratch[5].r + scratch[4].i; - Fout[m3].i = scratch[5].i - scratch[4].r; - }else{ - Fout[m].r = scratch[5].r + scratch[4].i; - Fout[m].i = scratch[5].i - scratch[4].r; - Fout[m3].r = scratch[5].r - scratch[4].i; - Fout[m3].i = scratch[5].i + scratch[4].r; - } - ++Fout; - }while(--k); -} - -static void kf_bfly3( - kiss_fft_cpx * Fout, - const size_t fstride, - const kiss_fft_cfg st, - size_t m - ) -{ - size_t k=m; - const size_t m2 = 2*m; - kiss_fft_cpx *tw1,*tw2; - kiss_fft_cpx scratch[5]; - kiss_fft_cpx epi3; - epi3 = st->twiddles[fstride*m]; - - tw1=tw2=st->twiddles; - - do{ - C_FIXDIV(*Fout,3); C_FIXDIV(Fout[m],3); C_FIXDIV(Fout[m2],3); - - C_MUL(scratch[1],Fout[m] , *tw1); - C_MUL(scratch[2],Fout[m2] , *tw2); - - C_ADD(scratch[3],scratch[1],scratch[2]); - C_SUB(scratch[0],scratch[1],scratch[2]); - tw1 += fstride; - tw2 += fstride*2; - - Fout[m].r = Fout->r - HALF_OF(scratch[3].r); - Fout[m].i = Fout->i - HALF_OF(scratch[3].i); - - C_MULBYSCALAR( scratch[0] , epi3.i ); - - C_ADDTO(*Fout,scratch[3]); - - Fout[m2].r = Fout[m].r + scratch[0].i; - Fout[m2].i = Fout[m].i - scratch[0].r; - - Fout[m].r -= scratch[0].i; - Fout[m].i += scratch[0].r; - - ++Fout; - }while(--k); -} - -static void kf_bfly5( - kiss_fft_cpx * Fout, - const size_t fstride, - const kiss_fft_cfg st, - int m - ) -{ - kiss_fft_cpx *Fout0,*Fout1,*Fout2,*Fout3,*Fout4; - int u; - kiss_fft_cpx scratch[13]; - kiss_fft_cpx * twiddles = st->twiddles; - kiss_fft_cpx *tw; - kiss_fft_cpx ya,yb; - ya = twiddles[fstride*m]; - yb = twiddles[fstride*2*m]; - - Fout0=Fout; - Fout1=Fout0+m; - Fout2=Fout0+2*m; - Fout3=Fout0+3*m; - Fout4=Fout0+4*m; - - tw=st->twiddles; - for ( u=0; ur += scratch[7].r + scratch[8].r; - Fout0->i += scratch[7].i + scratch[8].i; - - scratch[5].r = scratch[0].r + S_MUL(scratch[7].r,ya.r) + S_MUL(scratch[8].r,yb.r); - scratch[5].i = scratch[0].i + S_MUL(scratch[7].i,ya.r) + S_MUL(scratch[8].i,yb.r); - - scratch[6].r = S_MUL(scratch[10].i,ya.i) + S_MUL(scratch[9].i,yb.i); - scratch[6].i = -S_MUL(scratch[10].r,ya.i) - S_MUL(scratch[9].r,yb.i); - - C_SUB(*Fout1,scratch[5],scratch[6]); - C_ADD(*Fout4,scratch[5],scratch[6]); - - scratch[11].r = scratch[0].r + S_MUL(scratch[7].r,yb.r) + S_MUL(scratch[8].r,ya.r); - scratch[11].i = scratch[0].i + S_MUL(scratch[7].i,yb.r) + S_MUL(scratch[8].i,ya.r); - scratch[12].r = - S_MUL(scratch[10].i,yb.i) + S_MUL(scratch[9].i,ya.i); - scratch[12].i = S_MUL(scratch[10].r,yb.i) - S_MUL(scratch[9].r,ya.i); - - C_ADD(*Fout2,scratch[11],scratch[12]); - C_SUB(*Fout3,scratch[11],scratch[12]); - - ++Fout0;++Fout1;++Fout2;++Fout3;++Fout4; - } -} - -/* perform the butterfly for one stage of a mixed radix FFT */ -static void kf_bfly_generic( - kiss_fft_cpx * Fout, - const size_t fstride, - const kiss_fft_cfg st, - int m, - int p - ) -{ - int u,k,q1,q; - kiss_fft_cpx * twiddles = st->twiddles; - kiss_fft_cpx t; - int Norig = st->nfft; - - kiss_fft_cpx * scratch = (kiss_fft_cpx*)KISS_FFT_TMP_ALLOC(sizeof(kiss_fft_cpx)*p); - - for ( u=0; u=Norig) twidx-=Norig; - C_MUL(t,scratch[q] , twiddles[twidx] ); - C_ADDTO( Fout[ k ] ,t); - } - k += m; - } - } - KISS_FFT_TMP_FREE(scratch); -} - -static -void kf_work( - kiss_fft_cpx * Fout, - const kiss_fft_cpx * f, - const size_t fstride, - int in_stride, - int * factors, - const kiss_fft_cfg st - ) -{ - kiss_fft_cpx * Fout_beg=Fout; - const int p=*factors++; /* the radix */ - const int m=*factors++; /* stage's fft length/p */ - const kiss_fft_cpx * Fout_end = Fout + p*m; - -#ifdef _OPENMP - // use openmp extensions at the - // top-level (not recursive) - if (fstride==1 && p<=5) - { - int k; - - // execute the p different work units in different threads -# pragma omp parallel for - for (k=0;k floor_sqrt) - p = n; /* no more factors, skip to end */ - } - n /= p; - *facbuf++ = p; - *facbuf++ = n; - } while (n > 1); -} - -/* - * - * User-callable function to allocate all necessary storage space for the fft. - * - * The return value is a contiguous block of memory, allocated with malloc. As such, - * It can be freed with free(), rather than a kiss_fft-specific function. - * */ -kiss_fft_cfg kiss_fft_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem ) -{ - kiss_fft_cfg st=NULL; - size_t memneeded = sizeof(struct kiss_fft_state) - + sizeof(kiss_fft_cpx)*(nfft-1); /* twiddle factors*/ - - if ( lenmem==NULL ) { - st = ( kiss_fft_cfg)KISS_FFT_MALLOC( memneeded ); - }else{ - if (mem != NULL && *lenmem >= memneeded) - st = (kiss_fft_cfg)mem; - *lenmem = memneeded; - } - if (st) { - int i; - st->nfft=nfft; - st->inverse = inverse_fft; - - for (i=0;iinverse) - phase *= -1; - kf_cexp(st->twiddles+i, phase ); - } - - kf_factor(nfft,st->factors); - } - return st; -} - - -void kiss_fft_stride(kiss_fft_cfg st,const kiss_fft_cpx *fin,kiss_fft_cpx *fout,int in_stride) -{ - if (fin == fout) { - //NOTE: this is not really an in-place FFT algorithm. - //It just performs an out-of-place FFT into a temp buffer - kiss_fft_cpx * tmpbuf = (kiss_fft_cpx*)KISS_FFT_TMP_ALLOC( sizeof(kiss_fft_cpx)*st->nfft); - kf_work(tmpbuf,fin,1,in_stride, st->factors,st); - memcpy(fout,tmpbuf,sizeof(kiss_fft_cpx)*st->nfft); - KISS_FFT_TMP_FREE(tmpbuf); - }else{ - kf_work( fout, fin, 1,in_stride, st->factors,st ); - } -} - -void kiss_fft(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout) -{ - kiss_fft_stride(cfg,fin,fout,1); -} - - -void kiss_fft_cleanup(void) -{ - // nothing needed any more -} - -int kiss_fft_next_fast_size(int n) -{ - while(1) { - int m=n; - while ( (m%2) == 0 ) m/=2; - while ( (m%3) == 0 ) m/=3; - while ( (m%5) == 0 ) m/=5; - if (m<=1) - break; /* n is completely factorable by twos, threes, and fives */ - n++; - } - return n; -} diff --git a/native_client/kiss_fft130/kiss_fft.h b/native_client/kiss_fft130/kiss_fft.h deleted file mode 100644 index 64c50f4aae..0000000000 --- a/native_client/kiss_fft130/kiss_fft.h +++ /dev/null @@ -1,124 +0,0 @@ -#ifndef KISS_FFT_H -#define KISS_FFT_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - ATTENTION! - If you would like a : - -- a utility that will handle the caching of fft objects - -- real-only (no imaginary time component ) FFT - -- a multi-dimensional FFT - -- a command-line utility to perform ffts - -- a command-line utility to perform fast-convolution filtering - - Then see kfc.h kiss_fftr.h kiss_fftnd.h fftutil.c kiss_fastfir.c - in the tools/ directory. -*/ - -#ifdef USE_SIMD -# include -# define kiss_fft_scalar __m128 -#define KISS_FFT_MALLOC(nbytes) _mm_malloc(nbytes,16) -#define KISS_FFT_FREE _mm_free -#else -#define KISS_FFT_MALLOC malloc -#define KISS_FFT_FREE free -#endif - - -#ifdef FIXED_POINT -#include -# if (FIXED_POINT == 32) -# define kiss_fft_scalar int32_t -# else -# define kiss_fft_scalar int16_t -# endif -#else -# ifndef kiss_fft_scalar -/* default is float */ -# define kiss_fft_scalar float -# endif -#endif - -typedef struct { - kiss_fft_scalar r; - kiss_fft_scalar i; -}kiss_fft_cpx; - -typedef struct kiss_fft_state* kiss_fft_cfg; - -/* - * kiss_fft_alloc - * - * Initialize a FFT (or IFFT) algorithm's cfg/state buffer. - * - * typical usage: kiss_fft_cfg mycfg=kiss_fft_alloc(1024,0,NULL,NULL); - * - * The return value from fft_alloc is a cfg buffer used internally - * by the fft routine or NULL. - * - * If lenmem is NULL, then kiss_fft_alloc will allocate a cfg buffer using malloc. - * The returned value should be free()d when done to avoid memory leaks. - * - * The state can be placed in a user supplied buffer 'mem': - * If lenmem is not NULL and mem is not NULL and *lenmem is large enough, - * then the function places the cfg in mem and the size used in *lenmem - * and returns mem. - * - * If lenmem is not NULL and ( mem is NULL or *lenmem is not large enough), - * then the function returns NULL and places the minimum cfg - * buffer size in *lenmem. - * */ - -kiss_fft_cfg kiss_fft_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem); - -/* - * kiss_fft(cfg,in_out_buf) - * - * Perform an FFT on a complex input buffer. - * for a forward FFT, - * fin should be f[0] , f[1] , ... ,f[nfft-1] - * fout will be F[0] , F[1] , ... ,F[nfft-1] - * Note that each element is complex and can be accessed like - f[k].r and f[k].i - * */ -void kiss_fft(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); - -/* - A more generic version of the above function. It reads its input from every Nth sample. - * */ -void kiss_fft_stride(kiss_fft_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout,int fin_stride); - -/* If kiss_fft_alloc allocated a buffer, it is one contiguous - buffer and can be simply free()d when no longer needed*/ -#define kiss_fft_free free - -/* - Cleans up some memory that gets managed internally. Not necessary to call, but it might clean up - your compiler output to call this before you exit. -*/ -void kiss_fft_cleanup(void); - - -/* - * Returns the smallest integer k, such that k>=n and k has only "fast" factors (2,3,5) - */ -int kiss_fft_next_fast_size(int n); - -/* for real ffts, we need an even size */ -#define kiss_fftr_next_fast_size_real(n) \ - (kiss_fft_next_fast_size( ((n)+1)>>1)<<1) - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/native_client/kiss_fft130/kissfft.hh b/native_client/kiss_fft130/kissfft.hh deleted file mode 100644 index a586cb1154..0000000000 --- a/native_client/kiss_fft130/kissfft.hh +++ /dev/null @@ -1,299 +0,0 @@ -#ifndef KISSFFT_CLASS_HH -#include -#include - -namespace kissfft_utils { - -template -struct traits -{ - typedef T_scalar scalar_type; - typedef std::complex cpx_type; - void fill_twiddles( std::complex * dst ,int nfft,bool inverse) - { - T_scalar phinc = (inverse?2:-2)* acos( (T_scalar) -1) / nfft; - for (int i=0;i(0,i*phinc) ); - } - - void prepare( - std::vector< std::complex > & dst, - int nfft,bool inverse, - std::vector & stageRadix, - std::vector & stageRemainder ) - { - _twiddles.resize(nfft); - fill_twiddles( &_twiddles[0],nfft,inverse); - dst = _twiddles; - - //factorize - //start factoring out 4's, then 2's, then 3,5,7,9,... - int n= nfft; - int p=4; - do { - while (n % p) { - switch (p) { - case 4: p = 2; break; - case 2: p = 3; break; - default: p += 2; break; - } - if (p*p>n) - p=n;// no more factors - } - n /= p; - stageRadix.push_back(p); - stageRemainder.push_back(n); - }while(n>1); - } - std::vector _twiddles; - - - const cpx_type twiddle(int i) { return _twiddles[i]; } -}; - -} - -template - > -class kissfft -{ - public: - typedef T_traits traits_type; - typedef typename traits_type::scalar_type scalar_type; - typedef typename traits_type::cpx_type cpx_type; - - kissfft(int nfft,bool inverse,const traits_type & traits=traits_type() ) - :_nfft(nfft),_inverse(inverse),_traits(traits) - { - _traits.prepare(_twiddles, _nfft,_inverse ,_stageRadix, _stageRemainder); - } - - void transform(const cpx_type * src , cpx_type * dst) - { - kf_work(0, dst, src, 1,1); - } - - private: - void kf_work( int stage,cpx_type * Fout, const cpx_type * f, size_t fstride,size_t in_stride) - { - int p = _stageRadix[stage]; - int m = _stageRemainder[stage]; - cpx_type * Fout_beg = Fout; - cpx_type * Fout_end = Fout + p*m; - - if (m==1) { - do{ - *Fout = *f; - f += fstride*in_stride; - }while(++Fout != Fout_end ); - }else{ - do{ - // recursive call: - // DFT of size m*p performed by doing - // p instances of smaller DFTs of size m, - // each one takes a decimated version of the input - kf_work(stage+1, Fout , f, fstride*p,in_stride); - f += fstride*in_stride; - }while( (Fout += m) != Fout_end ); - } - - Fout=Fout_beg; - - // recombine the p smaller DFTs - switch (p) { - case 2: kf_bfly2(Fout,fstride,m); break; - case 3: kf_bfly3(Fout,fstride,m); break; - case 4: kf_bfly4(Fout,fstride,m); break; - case 5: kf_bfly5(Fout,fstride,m); break; - default: kf_bfly_generic(Fout,fstride,m,p); break; - } - } - - // these were #define macros in the original kiss_fft - void C_ADD( cpx_type & c,const cpx_type & a,const cpx_type & b) { c=a+b;} - void C_MUL( cpx_type & c,const cpx_type & a,const cpx_type & b) { c=a*b;} - void C_SUB( cpx_type & c,const cpx_type & a,const cpx_type & b) { c=a-b;} - void C_ADDTO( cpx_type & c,const cpx_type & a) { c+=a;} - void C_FIXDIV( cpx_type & ,int ) {} // NO-OP for float types - scalar_type S_MUL( const scalar_type & a,const scalar_type & b) { return a*b;} - scalar_type HALF_OF( const scalar_type & a) { return a*.5;} - void C_MULBYSCALAR(cpx_type & c,const scalar_type & a) {c*=a;} - - void kf_bfly2( cpx_type * Fout, const size_t fstride, int m) - { - for (int k=0;kreal() - HALF_OF(scratch[3].real() ) , Fout->imag() - HALF_OF(scratch[3].imag() ) ); - - C_MULBYSCALAR( scratch[0] , epi3.imag() ); - - C_ADDTO(*Fout,scratch[3]); - - Fout[m2] = cpx_type( Fout[m].real() + scratch[0].imag() , Fout[m].imag() - scratch[0].real() ); - - C_ADDTO( Fout[m] , cpx_type( -scratch[0].imag(),scratch[0].real() ) ); - ++Fout; - }while(--k); - } - - void kf_bfly5( cpx_type * Fout, const size_t fstride, const size_t m) - { - cpx_type *Fout0,*Fout1,*Fout2,*Fout3,*Fout4; - size_t u; - cpx_type scratch[13]; - cpx_type * twiddles = &_twiddles[0]; - cpx_type *tw; - cpx_type ya,yb; - ya = twiddles[fstride*m]; - yb = twiddles[fstride*2*m]; - - Fout0=Fout; - Fout1=Fout0+m; - Fout2=Fout0+2*m; - Fout3=Fout0+3*m; - Fout4=Fout0+4*m; - - tw=twiddles; - for ( u=0; u=Norig) twidx-=Norig; - C_MUL(t,scratchbuf[q] , twiddles[twidx] ); - C_ADDTO( Fout[ k ] ,t); - } - k += m; - } - } - } - - int _nfft; - bool _inverse; - std::vector _twiddles; - std::vector _stageRadix; - std::vector _stageRemainder; - traits_type _traits; -}; -#endif diff --git a/native_client/kiss_fft130/test/Makefile b/native_client/kiss_fft130/test/Makefile deleted file mode 100644 index c204511ea4..0000000000 --- a/native_client/kiss_fft130/test/Makefile +++ /dev/null @@ -1,108 +0,0 @@ - -WARNINGS=-W -Wall -Wstrict-prototypes -Wmissing-prototypes -Waggregate-return \ - -Wcast-align -Wcast-qual -Wnested-externs -Wshadow -Wbad-function-cast \ - -Wwrite-strings - -CFLAGS=-O3 -I.. -I../tools $(WARNINGS) -CFLAGS+=-ffast-math -fomit-frame-pointer -#CFLAGS+=-funroll-loops -#CFLAGS+=-march=prescott -#CFLAGS+= -mtune=native -# TIP: try adding -openmp or -fopenmp to enable OPENMP directives and use of multiple cores -#CFLAGS+=-fopenmp -CFLAGS+= $(CFLAGADD) - - -ifeq "$(NFFT)" "" - NFFT=1800 -endif -ifeq "$(NUMFFTS)" "" - NUMFFTS=10000 -endif - -ifeq "$(DATATYPE)" "" - DATATYPE=float -endif - -BENCHKISS=bm_kiss_$(DATATYPE) -BENCHFFTW=bm_fftw_$(DATATYPE) -SELFTEST=st_$(DATATYPE) -TESTREAL=tr_$(DATATYPE) -TESTKFC=tkfc_$(DATATYPE) -FASTFILTREAL=ffr_$(DATATYPE) -SELFTESTSRC=twotonetest.c - - -TYPEFLAGS=-Dkiss_fft_scalar=$(DATATYPE) - -ifeq "$(DATATYPE)" "int16_t" - TYPEFLAGS=-DFIXED_POINT=16 -endif - -ifeq "$(DATATYPE)" "int32_t" - TYPEFLAGS=-DFIXED_POINT=32 -endif - -ifeq "$(DATATYPE)" "simd" - TYPEFLAGS=-DUSE_SIMD=1 -msse -endif - - -ifeq "$(DATATYPE)" "float" - # fftw needs to be built with --enable-float to build this lib - FFTWLIB=-lfftw3f -else - FFTWLIB=-lfftw3 -endif - -FFTWLIBDIR=-L/usr/local/lib/ - -SRCFILES=../kiss_fft.c ../tools/kiss_fftnd.c ../tools/kiss_fftr.c pstats.c ../tools/kfc.c ../tools/kiss_fftndr.c - -all: tools $(BENCHKISS) $(SELFTEST) $(BENCHFFTW) $(TESTREAL) $(TESTKFC) - -tools: - cd ../tools && make all - - -$(SELFTEST): $(SELFTESTSRC) $(SRCFILES) - $(CC) -o $@ $(CFLAGS) $(TYPEFLAGS) $+ -lm - -$(TESTKFC): $(SRCFILES) - $(CC) -o $@ $(CFLAGS) -DKFC_TEST $(TYPEFLAGS) $+ -lm - -$(TESTREAL): test_real.c $(SRCFILES) - $(CC) -o $@ $(CFLAGS) $(TYPEFLAGS) $+ -lm - -$(BENCHKISS): benchkiss.c $(SRCFILES) - $(CC) -o $@ $(CFLAGS) $(TYPEFLAGS) $+ -lm - -$(BENCHFFTW): benchfftw.c pstats.c - @echo "======attempting to build FFTW benchmark" - @$(CC) -o $@ $(CFLAGS) -DDATATYPE$(DATATYPE) $+ $(FFTWLIB) $(FFTWLIBDIR) -lm || echo "FFTW not available for comparison" - -test: all - @./$(TESTKFC) - @echo "======1d & 2-d complex fft self test (type= $(DATATYPE) )" - @./$(SELFTEST) - @echo "======real FFT (type= $(DATATYPE) )" - @./$(TESTREAL) - @echo "======timing test (type=$(DATATYPE))" - @./$(BENCHKISS) -x $(NUMFFTS) -n $(NFFT) - @[ -x ./$(BENCHFFTW) ] && ./$(BENCHFFTW) -x $(NUMFFTS) -n $(NFFT) ||true - @echo "======higher dimensions type=$(DATATYPE))" - @./testkiss.py - -selftest.c: - ./mk_test.py 10 12 14 > selftest.c -selftest_short.c: - ./mk_test.py -s 10 12 14 > selftest_short.c - - -CXXFLAGS=-O3 -ffast-math -fomit-frame-pointer -I.. -I../tools -W -Wall -testcpp: testcpp.cc ../kissfft.hh - $(CXX) -o $@ $(CXXFLAGS) testcpp.cc -lm - - -clean: - rm -f *~ bm_* st_* tr_* kf_* tkfc_* ff_* ffr_* *.pyc *.pyo *.dat testcpp diff --git a/native_client/kiss_fft130/test/benchfftw.c b/native_client/kiss_fft130/test/benchfftw.c deleted file mode 100644 index 8824d19528..0000000000 --- a/native_client/kiss_fft130/test/benchfftw.c +++ /dev/null @@ -1,94 +0,0 @@ -#include -#include -#include -#include -#include "pstats.h" - -#ifdef DATATYPEdouble - -#define CPXTYPE fftw_complex -#define PLAN fftw_plan -#define FFTMALLOC fftw_malloc -#define MAKEPLAN fftw_plan_dft_1d -#define DOFFT fftw_execute -#define DESTROYPLAN fftw_destroy_plan -#define FFTFREE fftw_free - -#elif defined(DATATYPEfloat) - -#define CPXTYPE fftwf_complex -#define PLAN fftwf_plan -#define FFTMALLOC fftwf_malloc -#define MAKEPLAN fftwf_plan_dft_1d -#define DOFFT fftwf_execute -#define DESTROYPLAN fftwf_destroy_plan -#define FFTFREE fftwf_free - -#endif - -#ifndef CPXTYPE -int main(void) -{ - fprintf(stderr,"Datatype not available in FFTW\n" ); - return 0; -} -#else -int main(int argc,char ** argv) -{ - int nfft=1024; - int isinverse=0; - int numffts=1000,i; - - CPXTYPE * in=NULL; - CPXTYPE * out=NULL; - PLAN p; - - pstats_init(); - - while (1) { - int c = getopt (argc, argv, "n:ix:h"); - if (c == -1) - break; - switch (c) { - case 'n': - nfft = atoi (optarg); - break; - case 'x': - numffts = atoi (optarg); - break; - case 'i': - isinverse = 1; - break; - case 'h': - case '?': - default: - fprintf(stderr,"options:\n-n N: complex fft length\n-i: inverse\n-x N: number of ffts to compute\n" - ""); - } - } - - in=FFTMALLOC(sizeof(CPXTYPE) * nfft); - out=FFTMALLOC(sizeof(CPXTYPE) * nfft); - for (i=0;i -#include -#include -#include -#include "kiss_fft.h" -#include "kiss_fftr.h" -#include "kiss_fftnd.h" -#include "kiss_fftndr.h" - -#include "pstats.h" - -static -int getdims(int * dims, char * arg) -{ - char *s; - int ndims=0; - while ( (s=strtok( arg , ",") ) ) { - dims[ndims++] = atoi(s); - //printf("%s=%d\n",s,dims[ndims-1]); - arg=NULL; - } - return ndims; -} - -int main(int argc,char ** argv) -{ - int k; - int nfft[32]; - int ndims = 1; - int isinverse=0; - int numffts=1000,i; - kiss_fft_cpx * buf; - kiss_fft_cpx * bufout; - int real = 0; - - nfft[0] = 1024;// default - - while (1) { - int c = getopt (argc, argv, "n:ix:r"); - if (c == -1) - break; - switch (c) { - case 'r': - real = 1; - break; - case 'n': - ndims = getdims(nfft, optarg ); - if (nfft[0] != kiss_fft_next_fast_size(nfft[0]) ) { - int ng = kiss_fft_next_fast_size(nfft[0]); - fprintf(stderr,"warning: %d might be a better choice for speed than %d\n",ng,nfft[0]); - } - break; - case 'x': - numffts = atoi (optarg); - break; - case 'i': - isinverse = 1; - break; - } - } - int nbytes = sizeof(kiss_fft_cpx); - for (k=0;k - -#include "kiss_fft.h" -#include "kiss_fftnd.h" -#include "kiss_fftr.h" - -BEGIN_BENCH_DOC -BENCH_DOC("name", "kissfft") -BENCH_DOC("version", "1.0.1") -BENCH_DOC("year", "2004") -BENCH_DOC("author", "Mark Borgerding") -BENCH_DOC("language", "C") -BENCH_DOC("url", "http://sourceforge.net/projects/kissfft/") -BENCH_DOC("copyright", -"Copyright (c) 2003,4 Mark Borgerding\n" -"\n" -"All rights reserved.\n" -"\n" -"Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:\n" -"\n" -" * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.\n" -" * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.\n" -" * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission.\n" -"\n" - "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n") -END_BENCH_DOC - -int can_do(struct problem *p) -{ - if (p->rank == 1) { - if (p->kind == PROBLEM_REAL) { - return (p->n[0] & 1) == 0; /* only even real is okay */ - } else { - return 1; - } - } else { - return p->kind == PROBLEM_COMPLEX; - } -} - -static kiss_fft_cfg cfg=NULL; -static kiss_fftr_cfg cfgr=NULL; -static kiss_fftnd_cfg cfgnd=NULL; - -#define FAILIF( c ) \ - if ( c ) do {\ - fprintf(stderr,\ - "kissfft: " #c " (file=%s:%d errno=%d %s)\n",\ - __FILE__,__LINE__ , errno,strerror( errno ) ) ;\ - exit(1);\ - }while(0) - - - -void setup(struct problem *p) -{ - size_t i; - - /* - fprintf(stderr,"%s %s %d-d ", - (p->sign == 1)?"Inverse":"Forward", - p->kind == PROBLEM_COMPLEX?"Complex":"Real", - p->rank); - */ - if (p->rank == 1) { - if (p->kind == PROBLEM_COMPLEX) { - cfg = kiss_fft_alloc (p->n[0], (p->sign == 1), 0, 0); - FAILIF(cfg==NULL); - }else{ - cfgr = kiss_fftr_alloc (p->n[0], (p->sign == 1), 0, 0); - FAILIF(cfgr==NULL); - } - }else{ - int dims[5]; - for (i=0;irank;++i){ - dims[i] = p->n[i]; - } - /* multi-dimensional */ - if (p->kind == PROBLEM_COMPLEX) { - cfgnd = kiss_fftnd_alloc( dims , p->rank, (p->sign == 1), 0, 0 ); - FAILIF(cfgnd==NULL); - } - } -} - -void doit(int iter, struct problem *p) -{ - int i; - void *in = p->in; - void *out = p->out; - - if (p->in_place) - out = p->in; - - if (p->rank == 1) { - if (p->kind == PROBLEM_COMPLEX){ - for (i = 0; i < iter; ++i) - kiss_fft (cfg, in, out); - } else { - /* PROBLEM_REAL */ - if (p->sign == -1) /* FORWARD */ - for (i = 0; i < iter; ++i) - kiss_fftr (cfgr, in, out); - else - for (i = 0; i < iter; ++i) - kiss_fftri (cfgr, in, out); - } - }else{ - /* multi-dimensional */ - for (i = 0; i < iter; ++i) - kiss_fftnd(cfgnd,in,out); - } -} - -void done(struct problem *p) -{ - free(cfg); - cfg=NULL; - free(cfgr); - cfgr=NULL; - free(cfgnd); - cfgnd=NULL; - UNUSED(p); -} diff --git a/native_client/kiss_fft130/test/fastfir.py b/native_client/kiss_fft130/test/fastfir.py deleted file mode 100755 index 5ff432a361..0000000000 --- a/native_client/kiss_fft130/test/fastfir.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - -from Numeric import * -from FFT import * - -def make_random(len): - import random - res=[] - for i in range(int(len)): - r=random.uniform(-1,1) - i=random.uniform(-1,1) - res.append( complex(r,i) ) - return res - -def slowfilter(sig,h): - translen = len(h)-1 - return convolve(sig,h)[translen:-translen] - -def nextpow2(x): - return 2 ** math.ceil(math.log(x)/math.log(2)) - -def fastfilter(sig,h,nfft=None): - if nfft is None: - nfft = int( nextpow2( 2*len(h) ) ) - H = fft( h , nfft ) - scraplen = len(h)-1 - keeplen = nfft-scraplen - res=[] - isdone = 0 - lastidx = nfft - idx0 = 0 - while not isdone: - idx1 = idx0 + nfft - if idx1 >= len(sig): - idx1 = len(sig) - lastidx = idx1-idx0 - if lastidx <= scraplen: - break - isdone = 1 - Fss = fft(sig[idx0:idx1],nfft) - fm = Fss * H - m = inverse_fft(fm) - res.append( m[scraplen:lastidx] ) - idx0 += keeplen - return concatenate( res ) - -def main(): - import sys - from getopt import getopt - opts,args = getopt(sys.argv[1:],'rn:l:') - opts=dict(opts) - - siglen = int(opts.get('-l',1e4 ) ) - hlen =50 - - nfft = int(opts.get('-n',128) ) - usereal = opts.has_key('-r') - - print 'nfft=%d'%nfft - # make a signal - sig = make_random( siglen ) - # make an impulse response - h = make_random( hlen ) - #h=[1]*2+[0]*3 - if usereal: - sig=[c.real for c in sig] - h=[c.real for c in h] - - # perform MAC filtering - yslow = slowfilter(sig,h) - #print '',yslow,'' - #yfast = fastfilter(sig,h,nfft) - yfast = utilfastfilter(sig,h,nfft,usereal) - #print yfast - print 'len(yslow)=%d'%len(yslow) - print 'len(yfast)=%d'%len(yfast) - diff = yslow-yfast - snr = 10*log10( abs( vdot(yslow,yslow) / vdot(diff,diff) ) ) - print 'snr=%s' % snr - if snr < 10.0: - print 'h=',h - print 'sig=',sig[:5],'...' - print 'yslow=',yslow[:5],'...' - print 'yfast=',yfast[:5],'...' - -def utilfastfilter(sig,h,nfft,usereal): - import compfft - import os - open( 'sig.dat','w').write( compfft.dopack(sig,'f',not usereal) ) - open( 'h.dat','w').write( compfft.dopack(h,'f',not usereal) ) - if usereal: - util = './fastconvr' - else: - util = './fastconv' - cmd = 'time %s -n %d -i sig.dat -h h.dat -o out.dat' % (util, nfft) - print cmd - ec = os.system(cmd) - print 'exited->',ec - return compfft.dounpack(open('out.dat').read(),'f',not usereal) - -if __name__ == "__main__": - main() diff --git a/native_client/kiss_fft130/test/fft.py b/native_client/kiss_fft130/test/fft.py deleted file mode 100755 index 2705f71ff7..0000000000 --- a/native_client/kiss_fft130/test/fft.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python - -import math -import sys -import random - -pi=math.pi -e=math.e -j=complex(0,1) - -def fft(f,inv): - n=len(f) - if n==1: - return f - - for p in 2,3,5: - if n%p==0: - break - else: - raise Exception('%s not factorable ' % n) - - m = n/p - Fout=[] - for q in range(p): # 0,1 - fp = f[q::p] # every p'th time sample - Fp = fft( fp ,inv) - Fout.extend( Fp ) - - for u in range(m): - scratch = Fout[u::m] # u to end in strides of m - for q1 in range(p): - k = q1*m + u # indices to Fout above that became scratch - Fout[ k ] = scratch[0] # cuz e**0==1 in loop below - for q in range(1,p): - if inv: - t = e ** ( j*2*pi*k*q/n ) - else: - t = e ** ( -j*2*pi*k*q/n ) - Fout[ k ] += scratch[q] * t - - return Fout - -def rifft(F): - N = len(F) - 1 - Z = [0] * (N) - for k in range(N): - Fek = ( F[k] + F[-k-1].conjugate() ) - Fok = ( F[k] - F[-k-1].conjugate() ) * e ** (j*pi*k/N) - Z[k] = Fek + j*Fok - - fp = fft(Z , 1) - - f = [] - for c in fp: - f.append(c.real) - f.append(c.imag) - return f - -def real_fft( f,inv ): - if inv: - return rifft(f) - - N = len(f) / 2 - - res = f[::2] - ims = f[1::2] - - fp = [ complex(r,i) for r,i in zip(res,ims) ] - print 'fft input ', fp - Fp = fft( fp ,0 ) - print 'fft output ', Fp - - F = [ complex(0,0) ] * ( N+1 ) - - F[0] = complex( Fp[0].real + Fp[0].imag , 0 ) - - for k in range(1,N/2+1): - tw = e ** ( -j*pi*(.5+float(k)/N ) ) - - F1k = Fp[k] + Fp[N-k].conjugate() - F2k = Fp[k] - Fp[N-k].conjugate() - F2k *= tw - F[k] = ( F1k + F2k ) * .5 - F[N-k] = ( F1k - F2k ).conjugate() * .5 - #F[N-k] = ( F1kp + e ** ( -j*pi*(.5+float(N-k)/N ) ) * F2kp ) * .5 - #F[N-k] = ( F1k.conjugate() - tw.conjugate() * F2k.conjugate() ) * .5 - - F[N] = complex( Fp[0].real - Fp[0].imag , 0 ) - return F - -def main(): - #fft_func = fft - fft_func = real_fft - - tvec = [0.309655,0.815653,0.768570,0.591841,0.404767,0.637617,0.007803,0.012665] - Ftvec = [ complex(r,i) for r,i in zip( - [3.548571,-0.378761,-0.061950,0.188537,-0.566981,0.188537,-0.061950,-0.378761], - [0.000000,-1.296198,-0.848764,0.225337,0.000000,-0.225337,0.848764,1.296198] ) ] - - F = fft_func( tvec,0 ) - - nerrs= 0 - for i in range(len(Ftvec)/2 + 1): - if abs( F[i] - Ftvec[i] )> 1e-5: - print 'F[%d]: %s != %s' % (i,F[i],Ftvec[i]) - nerrs += 1 - - print '%d errors in forward fft' % nerrs - if nerrs: - return - - trec = fft_func( F , 1 ) - - for i in range(len(trec) ): - trec[i] /= len(trec) - - for i in range(len(tvec) ): - if abs( trec[i] - tvec[i] )> 1e-5: - print 't[%d]: %s != %s' % (i,tvec[i],trec[i]) - nerrs += 1 - - print '%d errors in reverse fft' % nerrs - - -def make_random(dims=[1]): - import Numeric - res = [] - for i in range(dims[0]): - if len(dims)==1: - r=random.uniform(-1,1) - i=random.uniform(-1,1) - res.append( complex(r,i) ) - else: - res.append( make_random( dims[1:] ) ) - return Numeric.array(res) - -def flatten(x): - import Numeric - ntotal = Numeric.product(Numeric.shape(x)) - return Numeric.reshape(x,(ntotal,)) - -def randmat( ndims ): - dims=[] - for i in range( ndims ): - curdim = int( random.uniform(2,4) ) - dims.append( curdim ) - return make_random(dims ) - -def test_fftnd(ndims=3): - import FFT - import Numeric - - x=randmat( ndims ) - print 'dimensions=%s' % str( Numeric.shape(x) ) - #print 'x=%s' %str(x) - xver = FFT.fftnd(x) - x2=myfftnd(x) - err = xver - x2 - errf = flatten(err) - xverf = flatten(xver) - errpow = Numeric.vdot(errf,errf)+1e-10 - sigpow = Numeric.vdot(xverf,xverf)+1e-10 - snr = 10*math.log10(abs(sigpow/errpow) ) - if snr<80: - print xver - print x2 - print 'SNR=%sdB' % str( snr ) - -def myfftnd(x): - import Numeric - xf = flatten(x) - Xf = fftndwork( xf , Numeric.shape(x) ) - return Numeric.reshape(Xf,Numeric.shape(x) ) - -def fftndwork(x,dims): - import Numeric - dimprod=Numeric.product( dims ) - - for k in range( len(dims) ): - cur_dim=dims[ k ] - stride=dimprod/cur_dim - next_x = [complex(0,0)]*len(x) - for i in range(stride): - next_x[i*cur_dim:(i+1)*cur_dim] = fft(x[i:(i+cur_dim)*stride:stride],0) - x = next_x - return x - -if __name__ == "__main__": - try: - nd = int(sys.argv[1]) - except: - nd=None - if nd: - test_fftnd( nd ) - else: - sys.exit(0) diff --git a/native_client/kiss_fft130/test/mk_test.py b/native_client/kiss_fft130/test/mk_test.py deleted file mode 100755 index 998b730f51..0000000000 --- a/native_client/kiss_fft130/test/mk_test.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python - -import FFT -import sys -import random -import re -j=complex(0,1) - -def randvec(n,iscomplex): - if iscomplex: - return [ - int(random.uniform(-32768,32767) ) + j*int(random.uniform(-32768,32767) ) - for i in range(n) ] - else: - return [ int(random.uniform(-32768,32767) ) for i in range(n) ] - -def c_format(v,round=0): - if round: - return ','.join( [ '{%d,%d}' %(int(c.real),int(c.imag) ) for c in v ] ) - else: - s= ','.join( [ '{%.60f ,%.60f }' %(c.real,c.imag) for c in v ] ) - return re.sub(r'\.?0+ ',' ',s) - -def test_cpx( n,inverse ,short): - v = randvec(n,1) - scale = 1 - if short: - minsnr=30 - else: - minsnr=100 - - if inverse: - tvecout = FFT.inverse_fft(v) - if short: - scale = 1 - else: - scale = len(v) - else: - tvecout = FFT.fft(v) - if short: - scale = 1.0/len(v) - - tvecout = [ c * scale for c in tvecout ] - - - s="""#define NFFT %d""" % len(v) + """ - { - double snr; - kiss_fft_cpx test_vec_in[NFFT] = { """ + c_format(v) + """}; - kiss_fft_cpx test_vec_out[NFFT] = {""" + c_format( tvecout ) + """}; - kiss_fft_cpx testbuf[NFFT]; - void * cfg = kiss_fft_alloc(NFFT,%d,0,0);""" % inverse + """ - - kiss_fft(cfg,test_vec_in,testbuf); - snr = snr_compare(test_vec_out,testbuf,NFFT); - printf("DATATYPE=" xstr(kiss_fft_scalar) ", FFT n=%d, inverse=%d, snr = %g dB\\n",NFFT,""" + str(inverse) + """,snr); - if (snr<""" + str(minsnr) + """) - exit_code++; - free(cfg); - } -#undef NFFT -""" - return s - -def compare_func(): - s=""" -#define xstr(s) str(s) -#define str(s) #s -double snr_compare( kiss_fft_cpx * test_vec_out,kiss_fft_cpx * testbuf, int n) -{ - int k; - double sigpow,noisepow,err,snr,scale=0; - kiss_fft_cpx err; - sigpow = noisepow = .000000000000000000000000000001; - - for (k=0;k -#include -#include -#include -#include - -#include "pstats.h" - -static struct tms tms_beg; -static struct tms tms_end; -static int has_times = 0; - - -void pstats_init(void) -{ - has_times = times(&tms_beg) != -1; -} - -static void tms_report(void) -{ - double cputime; - if (! has_times ) - return; - times(&tms_end); - cputime = ( ((float)tms_end.tms_utime + tms_end.tms_stime + tms_end.tms_cutime + tms_end.tms_cstime ) - - ((float)tms_beg.tms_utime + tms_beg.tms_stime + tms_beg.tms_cutime + tms_beg.tms_cstime ) ) - / sysconf(_SC_CLK_TCK); - fprintf(stderr,"\tcputime=%.3f\n" , cputime); -} - -static void ps_report(void) -{ - char buf[1024]; -#ifdef __APPLE__ /* MAC OS X */ - sprintf(buf,"ps -o command,majflt,minflt,rss,pagein,vsz -p %d 1>&2",getpid() ); -#else /* GNU/Linux */ - sprintf(buf,"ps -o comm,majflt,minflt,rss,drs,pagein,sz,trs,vsz %d 1>&2",getpid() ); -#endif - if (system( buf )==-1) { - perror("system call to ps failed"); - } -} - -void pstats_report() -{ - ps_report(); - tms_report(); -} - diff --git a/native_client/kiss_fft130/test/pstats.h b/native_client/kiss_fft130/test/pstats.h deleted file mode 100644 index 71ff02a465..0000000000 --- a/native_client/kiss_fft130/test/pstats.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef PSTATS_H -#define PSTATS_H - -void pstats_init(void); -void pstats_report(void); - -#endif diff --git a/native_client/kiss_fft130/test/tailscrap.m b/native_client/kiss_fft130/test/tailscrap.m deleted file mode 100644 index abf904691c..0000000000 --- a/native_client/kiss_fft130/test/tailscrap.m +++ /dev/null @@ -1,26 +0,0 @@ -function maxabsdiff=tailscrap() -% test code for circular convolution with the scrapped portion -% at the tail of the buffer, rather than the front -% -% The idea is to rotate the zero-padded h (impulse response) buffer -% to the left nh-1 samples, rotating the junk samples as well. -% This could be very handy in avoiding buffer copies during fast filtering. -nh=10; -nfft=256; - -h=rand(1,nh); -x=rand(1,nfft); - -hpad=[ h(nh) zeros(1,nfft-nh) h(1:nh-1) ]; - -% baseline comparison -y1 = filter(h,1,x); -y1_notrans = y1(nh:nfft); - -% fast convolution -y2 = ifft( fft(hpad) .* fft(x) ); -y2_notrans=y2(1:nfft-nh+1); - -maxabsdiff = max(abs(y2_notrans - y1_notrans)) - -end diff --git a/native_client/kiss_fft130/test/test_real.c b/native_client/kiss_fft130/test/test_real.c deleted file mode 100644 index 36a0b086c6..0000000000 --- a/native_client/kiss_fft130/test/test_real.c +++ /dev/null @@ -1,172 +0,0 @@ -#include "kiss_fftr.h" -#include "_kiss_fft_guts.h" -#include -#include -#include - -static double cputime(void) -{ - struct tms t; - times(&t); - return (double)(t.tms_utime + t.tms_stime)/ sysconf(_SC_CLK_TCK) ; -} - -static -kiss_fft_scalar rand_scalar(void) -{ -#ifdef USE_SIMD - return _mm_set1_ps(rand()-RAND_MAX/2); -#else - kiss_fft_scalar s = (kiss_fft_scalar)(rand() -RAND_MAX/2); - return s/2; -#endif -} - -static -double snr_compare( kiss_fft_cpx * vec1,kiss_fft_cpx * vec2, int n) -{ - int k; - double sigpow=1e-10,noisepow=1e-10,err,snr,scale=0; - -#ifdef USE_SIMD - float *fv1 = (float*)vec1; - float *fv2 = (float*)vec2; - for (k=0;k<8*n;++k) { - sigpow += *fv1 * *fv1; - err = *fv1 - *fv2; - noisepow += err*err; - ++fv1; - ++fv2; - } -#else - for (k=0;k1) - nfft = atoi(argv[1]); - kiss_fft_cpx cin[nfft]; - kiss_fft_cpx cout[nfft]; - kiss_fft_cpx sout[nfft]; - kiss_fft_cfg kiss_fft_state; - kiss_fftr_cfg kiss_fftr_state; - - kiss_fft_scalar rin[nfft+2]; - kiss_fft_scalar rout[nfft+2]; - kiss_fft_scalar zero; - memset(&zero,0,sizeof(zero) ); // ugly way of setting short,int,float,double, or __m128 to zero - - srand(time(0)); - - for (i=0;i1) { - int k; - for (k=1;k -#include -#include - -#include -static inline -double curtime(void) -{ - struct timeval tv; - gettimeofday(&tv, NULL); - return (double)tv.tv_sec + (double)tv.tv_usec*.000001; -} - -using namespace std; - -template -void dotest(int nfft) -{ - typedef kissfft FFT; - typedef std::complex cpx_type; - - cout << "type:" << typeid(T).name() << " nfft:" << nfft; - - FFT fft(nfft,false); - - vector inbuf(nfft); - vector outbuf(nfft); - for (int k=0;k acc = 0; - long double phinc = 2*k0* M_PIl / nfft; - for (int k1=0;k1 x(inbuf[k1].real(),inbuf[k1].imag()); - acc += x * exp( complex(0,-k1*phinc) ); - } - totalpower += norm(acc); - complex x(outbuf[k0].real(),outbuf[k0].imag()); - complex dif = acc - x; - difpower += norm(dif); - } - cout << " RMSE:" << sqrt(difpower/totalpower) << "\t"; - - double t0 = curtime(); - int nits=20e6/nfft; - for (int k=0;k1) { - for (int k=1;k(nfft); dotest(nfft); dotest(nfft); - } - }else{ - dotest(32); dotest(32); dotest(32); - dotest(1024); dotest(1024); dotest(1024); - dotest(840); dotest(840); dotest(840); - } - return 0; -} diff --git a/native_client/kiss_fft130/test/testkiss.py b/native_client/kiss_fft130/test/testkiss.py deleted file mode 100755 index af75065450..0000000000 --- a/native_client/kiss_fft130/test/testkiss.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python - -import math -import sys -import os -import random -import struct -import popen2 -import getopt -import numpy - -pi=math.pi -e=math.e -j=complex(0,1) - -doreal=0 - -datatype = os.environ.get('DATATYPE','float') - -util = '../tools/fft_' + datatype -minsnr=90 -if datatype == 'double': - fmt='d' -elif datatype=='int16_t': - fmt='h' - minsnr=10 -elif datatype=='int32_t': - fmt='i' -elif datatype=='simd': - fmt='4f' - sys.stderr.write('testkiss.py does not yet test simd') - sys.exit(0) -elif datatype=='float': - fmt='f' -else: - sys.stderr.write('unrecognized datatype %s\n' % datatype) - sys.exit(1) - - -def dopack(x,cpx=1): - x = numpy.reshape( x, ( numpy.size(x),) ) - - if cpx: - s = ''.join( [ struct.pack(fmt*2,c.real,c.imag) for c in x ] ) - else: - s = ''.join( [ struct.pack(fmt,c.real) for c in x ] ) - return s - -def dounpack(x,cpx): - uf = fmt * ( len(x) / struct.calcsize(fmt) ) - s = struct.unpack(uf,x) - if cpx: - return numpy.array(s[::2]) + numpy.array( s[1::2] )*j - else: - return numpy.array(s ) - -def make_random(dims=[1]): - res = [] - for i in range(dims[0]): - if len(dims)==1: - r=random.uniform(-1,1) - if doreal: - res.append( r ) - else: - i=random.uniform(-1,1) - res.append( complex(r,i) ) - else: - res.append( make_random( dims[1:] ) ) - return numpy.array(res) - -def flatten(x): - ntotal = numpy.size(x) - return numpy.reshape(x,(ntotal,)) - -def randmat( ndims ): - dims=[] - for i in range( ndims ): - curdim = int( random.uniform(2,5) ) - if doreal and i==(ndims-1): - curdim = int(curdim/2)*2 # force even last dimension if real - dims.append( curdim ) - return make_random(dims ) - -def test_fft(ndims): - x=randmat( ndims ) - - - if doreal: - xver = numpy.fft.rfftn(x) - else: - xver = numpy.fft.fftn(x) - - open('/tmp/fftexp.dat','w').write(dopack( flatten(xver) , True ) ) - - x2=dofft(x,doreal) - err = xver - x2 - errf = flatten(err) - xverf = flatten(xver) - errpow = numpy.vdot(errf,errf)+1e-10 - sigpow = numpy.vdot(xverf,xverf)+1e-10 - snr = 10*math.log10(abs(sigpow/errpow) ) - print 'SNR (compared to NumPy) : %.1fdB' % float(snr) - - if snr -#include -#include -#include "kiss_fft.h" -#include "kiss_fftr.h" -#include - - -static -double two_tone_test( int nfft, int bin1,int bin2) -{ - kiss_fftr_cfg cfg = NULL; - kiss_fft_cpx *kout = NULL; - kiss_fft_scalar *tbuf = NULL; - - int i; - double f1 = bin1*2*M_PI/nfft; - double f2 = bin2*2*M_PI/nfft; - double sigpow=0; - double noisepow=0; -#if FIXED_POINT==32 - long maxrange = LONG_MAX; -#else - long maxrange = SHRT_MAX;/* works fine for float too*/ -#endif - - cfg = kiss_fftr_alloc(nfft , 0, NULL, NULL); - tbuf = KISS_FFT_MALLOC(nfft * sizeof(kiss_fft_scalar)); - kout = KISS_FFT_MALLOC(nfft * sizeof(kiss_fft_cpx)); - - /* generate a signal with two tones*/ - for (i = 0; i < nfft; i++) { -#ifdef USE_SIMD - tbuf[i] = _mm_set1_ps( (maxrange>>1)*cos(f1*i) - + (maxrange>>1)*cos(f2*i) ); -#else - tbuf[i] = (maxrange>>1)*cos(f1*i) - + (maxrange>>1)*cos(f2*i); -#endif - } - - kiss_fftr(cfg, tbuf, kout); - - for (i=0;i < (nfft/2+1);++i) { -#ifdef USE_SIMD - double tmpr = (double)*(float*)&kout[i].r / (double)maxrange; - double tmpi = (double)*(float*)&kout[i].i / (double)maxrange; -#else - double tmpr = (double)kout[i].r / (double)maxrange; - double tmpi = (double)kout[i].i / (double)maxrange; -#endif - double mag2 = tmpr*tmpr + tmpi*tmpi; - if (i!=0 && i!= nfft/2) - mag2 *= 2; /* all bins except DC and Nyquist have symmetric counterparts implied*/ - - /* if there is power in one of the expected bins, it is signal, otherwise noise*/ - if ( i!=bin1 && i != bin2 ) - noisepow += mag2; - else - sigpow += mag2; - } - kiss_fft_cleanup(); - /*printf("TEST %d,%d,%d noise @ %fdB\n",nfft,bin1,bin2,10*log10(noisepow/sigpow +1e-30) );*/ - return 10*log10(sigpow/(noisepow+1e-50) ); -} - -int main(int argc,char ** argv) -{ - int nfft = 4*2*2*3*5; - if (argc>1) nfft = atoi(argv[1]); - - int i,j; - double minsnr = 500; - double maxsnr = -500; - double snr; - for (i=0;i>4)+1) { - for (j=i;j>4)+7) { - snr = two_tone_test(nfft,i,j); - if (snrmaxsnr) { - maxsnr=snr; - } - } - } - snr = two_tone_test(nfft,nfft/2,nfft/2); - if (snrmaxsnr) maxsnr=snr; - - printf("TwoToneTest: snr ranges from %ddB to %ddB\n",(int)minsnr,(int)maxsnr); - printf("sizeof(kiss_fft_scalar) = %d\n",(int)sizeof(kiss_fft_scalar) ); - return 0; -} diff --git a/native_client/kiss_fft130/tools/Makefile b/native_client/kiss_fft130/tools/Makefile deleted file mode 100644 index ae7646b880..0000000000 --- a/native_client/kiss_fft130/tools/Makefile +++ /dev/null @@ -1,62 +0,0 @@ -WARNINGS=-W -Wall -Wstrict-prototypes -Wmissing-prototypes -Waggregate-return \ - -Wcast-align -Wcast-qual -Wnested-externs -Wshadow -Wbad-function-cast \ - -Wwrite-strings - -ifeq "$(DATATYPE)" "" - DATATYPE=float -endif - -ifeq "$(DATATYPE)" "int32_t" - TYPEFLAGS=-DFIXED_POINT=32 -endif - -ifeq "$(DATATYPE)" "int16_t" - TYPEFLAGS=-DFIXED_POINT=16 -endif - -ifeq "$(DATATYPE)" "simd" - TYPEFLAGS=-DUSE_SIMD=1 -msse -endif - -ifeq "$(TYPEFLAGS)" "" - TYPEFLAGS=-Dkiss_fft_scalar=$(DATATYPE) -endif - -ifneq ("$(KISS_FFT_USE_ALLOCA)","") - CFLAGS+= -DKISS_FFT_USE_ALLOCA=1 -endif -CFLAGS+= $(CFLAGADD) - - -FFTUTIL=fft_$(DATATYPE) -FASTFILT=fastconv_$(DATATYPE) -FASTFILTREAL=fastconvr_$(DATATYPE) -PSDPNG=psdpng_$(DATATYPE) -DUMPHDR=dumphdr_$(DATATYPE) - -all: $(FFTUTIL) $(FASTFILT) $(FASTFILTREAL) -# $(PSDPNG) -# $(DUMPHDR) - -#CFLAGS=-Wall -O3 -pedantic -march=pentiumpro -ffast-math -fomit-frame-pointer $(WARNINGS) -# If the above flags do not work, try the following -CFLAGS=-Wall -O3 $(WARNINGS) -# tip: try -openmp or -fopenmp to use multiple cores - -$(FASTFILTREAL): ../kiss_fft.c kiss_fastfir.c kiss_fftr.c - $(CC) -o $@ $(CFLAGS) -I.. $(TYPEFLAGS) -DREAL_FASTFIR $+ -DFAST_FILT_UTIL -lm - -$(FASTFILT): ../kiss_fft.c kiss_fastfir.c - $(CC) -o $@ $(CFLAGS) -I.. $(TYPEFLAGS) $+ -DFAST_FILT_UTIL -lm - -$(FFTUTIL): ../kiss_fft.c fftutil.c kiss_fftnd.c kiss_fftr.c kiss_fftndr.c - $(CC) -o $@ $(CFLAGS) -I.. $(TYPEFLAGS) $+ -lm - -$(PSDPNG): ../kiss_fft.c psdpng.c kiss_fftr.c - $(CC) -o $@ $(CFLAGS) -I.. $(TYPEFLAGS) $+ -lpng -lm - -$(DUMPHDR): ../kiss_fft.c dumphdr.c - $(CC) -o $@ $(CFLAGS) -I.. $(TYPEFLAGS) $+ -lm - -clean: - rm -f *~ fft fft_* fastconv fastconv_* fastconvr fastconvr_* psdpng psdpng_* diff --git a/native_client/kiss_fft130/tools/fftutil.c b/native_client/kiss_fft130/tools/fftutil.c deleted file mode 100644 index db5a815121..0000000000 --- a/native_client/kiss_fft130/tools/fftutil.c +++ /dev/null @@ -1,208 +0,0 @@ -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include -#include -#include -#include -#include - -#include "kiss_fft.h" -#include "kiss_fftndr.h" - -static -void fft_file(FILE * fin,FILE * fout,int nfft,int isinverse) -{ - kiss_fft_cfg st; - kiss_fft_cpx * buf; - kiss_fft_cpx * bufout; - - buf = (kiss_fft_cpx*)malloc(sizeof(kiss_fft_cpx) * nfft ); - bufout = (kiss_fft_cpx*)malloc(sizeof(kiss_fft_cpx) * nfft ); - st = kiss_fft_alloc( nfft ,isinverse ,0,0); - - while ( fread( buf , sizeof(kiss_fft_cpx) * nfft ,1, fin ) > 0 ) { - kiss_fft( st , buf ,bufout); - fwrite( bufout , sizeof(kiss_fft_cpx) , nfft , fout ); - } - free(st); - free(buf); - free(bufout); -} - -static -void fft_filend(FILE * fin,FILE * fout,int *dims,int ndims,int isinverse) -{ - kiss_fftnd_cfg st; - kiss_fft_cpx *buf; - int dimprod=1,i; - for (i=0;i 0) { - kiss_fftnd (st, buf, buf); - fwrite (buf, sizeof (kiss_fft_cpx), dimprod, fout); - } - free (st); - free (buf); -} - - - -static -void fft_filend_real(FILE * fin,FILE * fout,int *dims,int ndims,int isinverse) -{ - int dimprod=1,i; - kiss_fftndr_cfg st; - void *ibuf; - void *obuf; - int insize,outsize; // size in bytes - - for (i=0;i 0) { - if (isinverse) { - kiss_fftndri(st, - (kiss_fft_cpx*)ibuf, - (kiss_fft_scalar*)obuf); - }else{ - kiss_fftndr(st, - (kiss_fft_scalar*)ibuf, - (kiss_fft_cpx*)obuf); - } - fwrite (obuf, sizeof(kiss_fft_scalar), outsize,fout); - } - free(st); - free(ibuf); - free(obuf); -} - -static -void fft_file_real(FILE * fin,FILE * fout,int nfft,int isinverse) -{ - kiss_fftr_cfg st; - kiss_fft_scalar * rbuf; - kiss_fft_cpx * cbuf; - - rbuf = (kiss_fft_scalar*)malloc(sizeof(kiss_fft_scalar) * nfft ); - cbuf = (kiss_fft_cpx*)malloc(sizeof(kiss_fft_cpx) * (nfft/2+1) ); - st = kiss_fftr_alloc( nfft ,isinverse ,0,0); - - if (isinverse==0) { - while ( fread( rbuf , sizeof(kiss_fft_scalar) * nfft ,1, fin ) > 0 ) { - kiss_fftr( st , rbuf ,cbuf); - fwrite( cbuf , sizeof(kiss_fft_cpx) , (nfft/2 + 1) , fout ); - } - }else{ - while ( fread( cbuf , sizeof(kiss_fft_cpx) * (nfft/2+1) ,1, fin ) > 0 ) { - kiss_fftri( st , cbuf ,rbuf); - fwrite( rbuf , sizeof(kiss_fft_scalar) , nfft , fout ); - } - } - free(st); - free(rbuf); - free(cbuf); -} - -static -int get_dims(char * arg,int * dims) -{ - char *p0; - int ndims=0; - - do{ - p0 = strchr(arg,','); - if (p0) - *p0++ = '\0'; - dims[ndims++] = atoi(arg); -// fprintf(stderr,"dims[%d] = %d\n",ndims-1,dims[ndims-1]); - arg = p0; - }while (p0); - return ndims; -} - -int main(int argc,char ** argv) -{ - int isinverse=0; - int isreal=0; - FILE *fin=stdin; - FILE *fout=stdout; - int ndims=1; - int dims[32]; - dims[0] = 1024; /*default fft size*/ - - while (1) { - int c=getopt(argc,argv,"n:iR"); - if (c==-1) break; - switch (c) { - case 'n': - ndims = get_dims(optarg,dims); - break; - case 'i':isinverse=1;break; - case 'R':isreal=1;break; - case '?': - fprintf(stderr,"usage options:\n" - "\t-n d1[,d2,d3...]: fft dimension(s)\n" - "\t-i : inverse\n" - "\t-R : real input samples, not complex\n"); - exit (1); - default:fprintf(stderr,"bad %c\n",c);break; - } - } - - if ( optind < argc ) { - if (strcmp("-",argv[optind]) !=0) - fin = fopen(argv[optind],"rb"); - ++optind; - } - - if ( optind < argc ) { - if ( strcmp("-",argv[optind]) !=0 ) - fout = fopen(argv[optind],"wb"); - ++optind; - } - - if (ndims==1) { - if (isreal) - fft_file_real(fin,fout,dims[0],isinverse); - else - fft_file(fin,fout,dims[0],isinverse); - }else{ - if (isreal) - fft_filend_real(fin,fout,dims,ndims,isinverse); - else - fft_filend(fin,fout,dims,ndims,isinverse); - } - - if (fout!=stdout) fclose(fout); - if (fin!=stdin) fclose(fin); - - return 0; -} diff --git a/native_client/kiss_fft130/tools/kfc.c b/native_client/kiss_fft130/tools/kfc.c deleted file mode 100644 index d94d1240d6..0000000000 --- a/native_client/kiss_fft130/tools/kfc.c +++ /dev/null @@ -1,116 +0,0 @@ -#include "kfc.h" - -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - - -typedef struct cached_fft *kfc_cfg; - -struct cached_fft -{ - int nfft; - int inverse; - kiss_fft_cfg cfg; - kfc_cfg next; -}; - -static kfc_cfg cache_root=NULL; -static int ncached=0; - -static kiss_fft_cfg find_cached_fft(int nfft,int inverse) -{ - size_t len; - kfc_cfg cur=cache_root; - kfc_cfg prev=NULL; - while ( cur ) { - if ( cur->nfft == nfft && inverse == cur->inverse ) - break;/*found the right node*/ - prev = cur; - cur = prev->next; - } - if (cur== NULL) { - /* no cached node found, need to create a new one*/ - kiss_fft_alloc(nfft,inverse,0,&len); -#ifdef USE_SIMD - int padding = (16-sizeof(struct cached_fft)) & 15; - // make sure the cfg aligns on a 16 byte boundary - len += padding; -#endif - cur = (kfc_cfg)KISS_FFT_MALLOC((sizeof(struct cached_fft) + len )); - if (cur == NULL) - return NULL; - cur->cfg = (kiss_fft_cfg)(cur+1); -#ifdef USE_SIMD - cur->cfg = (kiss_fft_cfg) ((char*)(cur+1)+padding); -#endif - kiss_fft_alloc(nfft,inverse,cur->cfg,&len); - cur->nfft=nfft; - cur->inverse=inverse; - cur->next = NULL; - if ( prev ) - prev->next = cur; - else - cache_root = cur; - ++ncached; - } - return cur->cfg; -} - -void kfc_cleanup(void) -{ - kfc_cfg cur=cache_root; - kfc_cfg next=NULL; - while (cur){ - next = cur->next; - free(cur); - cur=next; - } - ncached=0; - cache_root = NULL; -} -void kfc_fft(int nfft, const kiss_fft_cpx * fin,kiss_fft_cpx * fout) -{ - kiss_fft( find_cached_fft(nfft,0),fin,fout ); -} - -void kfc_ifft(int nfft, const kiss_fft_cpx * fin,kiss_fft_cpx * fout) -{ - kiss_fft( find_cached_fft(nfft,1),fin,fout ); -} - -#ifdef KFC_TEST -static void check(int nc) -{ - if (ncached != nc) { - fprintf(stderr,"ncached should be %d,but it is %d\n",nc,ncached); - exit(1); - } -} - -int main(void) -{ - kiss_fft_cpx buf1[1024],buf2[1024]; - memset(buf1,0,sizeof(buf1)); - check(0); - kfc_fft(512,buf1,buf2); - check(1); - kfc_fft(512,buf1,buf2); - check(1); - kfc_ifft(512,buf1,buf2); - check(2); - kfc_cleanup(); - check(0); - return 0; -} -#endif diff --git a/native_client/kiss_fft130/tools/kfc.h b/native_client/kiss_fft130/tools/kfc.h deleted file mode 100644 index 9b5fd67737..0000000000 --- a/native_client/kiss_fft130/tools/kfc.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef KFC_H -#define KFC_H -#include "kiss_fft.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* -KFC -- Kiss FFT Cache - -Not needing to deal with kiss_fft_alloc and a config -object may be handy for a lot of programs. - -KFC uses the underlying KISS FFT functions, but caches the config object. -The first time kfc_fft or kfc_ifft for a given FFT size, the cfg -object is created for it. All subsequent calls use the cached -configuration object. - -NOTE: -You should probably not use this if your program will be using a lot -of various sizes of FFTs. There is a linear search through the -cached objects. If you are only using one or two FFT sizes, this -will be negligible. Otherwise, you may want to use another method -of managing the cfg objects. - - There is no automated cleanup of the cached objects. This could lead -to large memory usage in a program that uses a lot of *DIFFERENT* -sized FFTs. If you want to force all cached cfg objects to be freed, -call kfc_cleanup. - - */ - -/*forward complex FFT */ -void kfc_fft(int nfft, const kiss_fft_cpx * fin,kiss_fft_cpx * fout); -/*reverse complex FFT */ -void kfc_ifft(int nfft, const kiss_fft_cpx * fin,kiss_fft_cpx * fout); - -/*free all cached objects*/ -void kfc_cleanup(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/native_client/kiss_fft130/tools/kiss_fastfir.c b/native_client/kiss_fft130/tools/kiss_fastfir.c deleted file mode 100644 index 4560aa3793..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fastfir.c +++ /dev/null @@ -1,470 +0,0 @@ -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "_kiss_fft_guts.h" - - -/* - Some definitions that allow real or complex filtering -*/ -#ifdef REAL_FASTFIR -#define MIN_FFT_LEN 2048 -#include "kiss_fftr.h" -typedef kiss_fft_scalar kffsamp_t; -typedef kiss_fftr_cfg kfcfg_t; -#define FFT_ALLOC kiss_fftr_alloc -#define FFTFWD kiss_fftr -#define FFTINV kiss_fftri -#else -#define MIN_FFT_LEN 1024 -typedef kiss_fft_cpx kffsamp_t; -typedef kiss_fft_cfg kfcfg_t; -#define FFT_ALLOC kiss_fft_alloc -#define FFTFWD kiss_fft -#define FFTINV kiss_fft -#endif - -typedef struct kiss_fastfir_state *kiss_fastfir_cfg; - - - -kiss_fastfir_cfg kiss_fastfir_alloc(const kffsamp_t * imp_resp,size_t n_imp_resp, - size_t * nfft,void * mem,size_t*lenmem); - -/* see do_file_filter for usage */ -size_t kiss_fastfir( kiss_fastfir_cfg cfg, kffsamp_t * inbuf, kffsamp_t * outbuf, size_t n, size_t *offset); - - - -static int verbose=0; - - -struct kiss_fastfir_state{ - size_t nfft; - size_t ngood; - kfcfg_t fftcfg; - kfcfg_t ifftcfg; - kiss_fft_cpx * fir_freq_resp; - kiss_fft_cpx * freqbuf; - size_t n_freq_bins; - kffsamp_t * tmpbuf; -}; - - -kiss_fastfir_cfg kiss_fastfir_alloc( - const kffsamp_t * imp_resp,size_t n_imp_resp, - size_t *pnfft, /* if <= 0, an appropriate size will be chosen */ - void * mem,size_t*lenmem) -{ - kiss_fastfir_cfg st = NULL; - size_t len_fftcfg,len_ifftcfg; - size_t memneeded = sizeof(struct kiss_fastfir_state); - char * ptr; - size_t i; - size_t nfft=0; - float scale; - int n_freq_bins; - if (pnfft) - nfft=*pnfft; - - if (nfft<=0) { - /* determine fft size as next power of two at least 2x - the impulse response length*/ - i=n_imp_resp-1; - nfft=2; - do{ - nfft<<=1; - }while (i>>=1); -#ifdef MIN_FFT_LEN - if ( nfft < MIN_FFT_LEN ) - nfft=MIN_FFT_LEN; -#endif - } - if (pnfft) - *pnfft = nfft; - -#ifdef REAL_FASTFIR - n_freq_bins = nfft/2 + 1; -#else - n_freq_bins = nfft; -#endif - /*fftcfg*/ - FFT_ALLOC (nfft, 0, NULL, &len_fftcfg); - memneeded += len_fftcfg; - /*ifftcfg*/ - FFT_ALLOC (nfft, 1, NULL, &len_ifftcfg); - memneeded += len_ifftcfg; - /* tmpbuf */ - memneeded += sizeof(kffsamp_t) * nfft; - /* fir_freq_resp */ - memneeded += sizeof(kiss_fft_cpx) * n_freq_bins; - /* freqbuf */ - memneeded += sizeof(kiss_fft_cpx) * n_freq_bins; - - if (lenmem == NULL) { - st = (kiss_fastfir_cfg) malloc (memneeded); - } else { - if (*lenmem >= memneeded) - st = (kiss_fastfir_cfg) mem; - *lenmem = memneeded; - } - if (!st) - return NULL; - - st->nfft = nfft; - st->ngood = nfft - n_imp_resp + 1; - st->n_freq_bins = n_freq_bins; - ptr=(char*)(st+1); - - st->fftcfg = (kfcfg_t)ptr; - ptr += len_fftcfg; - - st->ifftcfg = (kfcfg_t)ptr; - ptr += len_ifftcfg; - - st->tmpbuf = (kffsamp_t*)ptr; - ptr += sizeof(kffsamp_t) * nfft; - - st->freqbuf = (kiss_fft_cpx*)ptr; - ptr += sizeof(kiss_fft_cpx) * n_freq_bins; - - st->fir_freq_resp = (kiss_fft_cpx*)ptr; - ptr += sizeof(kiss_fft_cpx) * n_freq_bins; - - FFT_ALLOC (nfft,0,st->fftcfg , &len_fftcfg); - FFT_ALLOC (nfft,1,st->ifftcfg , &len_ifftcfg); - - memset(st->tmpbuf,0,sizeof(kffsamp_t)*nfft); - /*zero pad in the middle to left-rotate the impulse response - This puts the scrap samples at the end of the inverse fft'd buffer */ - st->tmpbuf[0] = imp_resp[ n_imp_resp - 1 ]; - for (i=0;itmpbuf[ nfft - n_imp_resp + 1 + i ] = imp_resp[ i ]; - } - - FFTFWD(st->fftcfg,st->tmpbuf,st->fir_freq_resp); - - /* TODO: this won't work for fixed point */ - scale = 1.0 / st->nfft; - - for ( i=0; i < st->n_freq_bins; ++i ) { -#ifdef USE_SIMD - st->fir_freq_resp[i].r *= _mm_set1_ps(scale); - st->fir_freq_resp[i].i *= _mm_set1_ps(scale); -#else - st->fir_freq_resp[i].r *= scale; - st->fir_freq_resp[i].i *= scale; -#endif - } - return st; -} - -static void fastconv1buf(const kiss_fastfir_cfg st,const kffsamp_t * in,kffsamp_t * out) -{ - size_t i; - /* multiply the frequency response of the input signal by - that of the fir filter*/ - FFTFWD( st->fftcfg, in , st->freqbuf ); - for ( i=0; in_freq_bins; ++i ) { - kiss_fft_cpx tmpsamp; - C_MUL(tmpsamp,st->freqbuf[i],st->fir_freq_resp[i]); - st->freqbuf[i] = tmpsamp; - } - - /* perform the inverse fft*/ - FFTINV(st->ifftcfg,st->freqbuf,out); -} - -/* n : the size of inbuf and outbuf in samples - return value: the number of samples completely processed - n-retval samples should be copied to the front of the next input buffer */ -static size_t kff_nocopy( - kiss_fastfir_cfg st, - const kffsamp_t * inbuf, - kffsamp_t * outbuf, - size_t n) -{ - size_t norig=n; - while (n >= st->nfft ) { - fastconv1buf(st,inbuf,outbuf); - inbuf += st->ngood; - outbuf += st->ngood; - n -= st->ngood; - } - return norig - n; -} - -static -size_t kff_flush(kiss_fastfir_cfg st,const kffsamp_t * inbuf,kffsamp_t * outbuf,size_t n) -{ - size_t zpad=0,ntmp; - - ntmp = kff_nocopy(st,inbuf,outbuf,n); - n -= ntmp; - inbuf += ntmp; - outbuf += ntmp; - - zpad = st->nfft - n; - memset(st->tmpbuf,0,sizeof(kffsamp_t)*st->nfft ); - memcpy(st->tmpbuf,inbuf,sizeof(kffsamp_t)*n ); - - fastconv1buf(st,st->tmpbuf,st->tmpbuf); - - memcpy(outbuf,st->tmpbuf,sizeof(kffsamp_t)*( st->ngood - zpad )); - return ntmp + st->ngood - zpad; -} - -size_t kiss_fastfir( - kiss_fastfir_cfg vst, - kffsamp_t * inbuf, - kffsamp_t * outbuf, - size_t n_new, - size_t *offset) -{ - size_t ntot = n_new + *offset; - if (n_new==0) { - return kff_flush(vst,inbuf,outbuf,ntot); - }else{ - size_t nwritten = kff_nocopy(vst,inbuf,outbuf,ntot); - *offset = ntot - nwritten; - /*save the unused or underused samples at the front of the input buffer */ - memcpy( inbuf , inbuf+nwritten , *offset * sizeof(kffsamp_t) ); - return nwritten; - } -} - -#ifdef FAST_FILT_UTIL -#include -#include -#include -#include - -static -void direct_file_filter( - FILE * fin, - FILE * fout, - const kffsamp_t * imp_resp, - size_t n_imp_resp) -{ - size_t nlag = n_imp_resp - 1; - - const kffsamp_t *tmph; - kffsamp_t *buf, *circbuf; - kffsamp_t outval; - size_t nread; - size_t nbuf; - size_t oldestlag = 0; - size_t k, tap; -#ifndef REAL_FASTFIR - kffsamp_t tmp; -#endif - - nbuf = 4096; - buf = (kffsamp_t *) malloc ( sizeof (kffsamp_t) * nbuf); - circbuf = (kffsamp_t *) malloc (sizeof (kffsamp_t) * nlag); - if (!circbuf || !buf) { - perror("circbuf allocation"); - exit(1); - } - - if ( fread (circbuf, sizeof (kffsamp_t), nlag, fin) != nlag ) { - perror ("insufficient data to overcome transient"); - exit (1); - } - - do { - nread = fread (buf, sizeof (kffsamp_t), nbuf, fin); - if (nread <= 0) - break; - - for (k = 0; k < nread; ++k) { - tmph = imp_resp+nlag; -#ifdef REAL_FASTFIR -# ifdef USE_SIMD - outval = _mm_set1_ps(0); -#else - outval = 0; -#endif - for (tap = oldestlag; tap < nlag; ++tap) - outval += circbuf[tap] * *tmph--; - for (tap = 0; tap < oldestlag; ++tap) - outval += circbuf[tap] * *tmph--; - outval += buf[k] * *tmph; -#else -# ifdef USE_SIMD - outval.r = outval.i = _mm_set1_ps(0); -#else - outval.r = outval.i = 0; -#endif - for (tap = oldestlag; tap < nlag; ++tap){ - C_MUL(tmp,circbuf[tap],*tmph); - --tmph; - C_ADDTO(outval,tmp); - } - - for (tap = 0; tap < oldestlag; ++tap) { - C_MUL(tmp,circbuf[tap],*tmph); - --tmph; - C_ADDTO(outval,tmp); - } - C_MUL(tmp,buf[k],*tmph); - C_ADDTO(outval,tmp); -#endif - - circbuf[oldestlag++] = buf[k]; - buf[k] = outval; - - if (oldestlag == nlag) - oldestlag = 0; - } - - if (fwrite (buf, sizeof (buf[0]), nread, fout) != nread) { - perror ("short write"); - exit (1); - } - } while (nread); - free (buf); - free (circbuf); -} - -static -void do_file_filter( - FILE * fin, - FILE * fout, - const kffsamp_t * imp_resp, - size_t n_imp_resp, - size_t nfft ) -{ - int fdout; - size_t n_samps_buf; - - kiss_fastfir_cfg cfg; - kffsamp_t *inbuf,*outbuf; - int nread,nwrite; - size_t idx_inbuf; - - fdout = fileno(fout); - - cfg=kiss_fastfir_alloc(imp_resp,n_imp_resp,&nfft,0,0); - - /* use length to minimize buffer shift*/ - n_samps_buf = 8*4096/sizeof(kffsamp_t); - n_samps_buf = nfft + 4*(nfft-n_imp_resp+1); - - if (verbose) fprintf(stderr,"bufsize=%d\n",(int)(sizeof(kffsamp_t)*n_samps_buf) ); - - - /*allocate space and initialize pointers */ - inbuf = (kffsamp_t*)malloc(sizeof(kffsamp_t)*n_samps_buf); - outbuf = (kffsamp_t*)malloc(sizeof(kffsamp_t)*n_samps_buf); - - idx_inbuf=0; - do{ - /* start reading at inbuf[idx_inbuf] */ - nread = fread( inbuf + idx_inbuf, sizeof(kffsamp_t), n_samps_buf - idx_inbuf,fin ); - - /* If nread==0, then this is a flush. - The total number of samples in input is idx_inbuf + nread . */ - nwrite = kiss_fastfir(cfg, inbuf, outbuf,nread,&idx_inbuf) * sizeof(kffsamp_t); - /* kiss_fastfir moved any unused samples to the front of inbuf and updated idx_inbuf */ - - if ( write(fdout, outbuf, nwrite) != nwrite ) { - perror("short write"); - exit(1); - } - }while ( nread ); - free(cfg); - free(inbuf); - free(outbuf); -} - -int main(int argc,char**argv) -{ - kffsamp_t * h; - int use_direct=0; - size_t nh,nfft=0; - FILE *fin=stdin; - FILE *fout=stdout; - FILE *filtfile=NULL; - while (1) { - int c=getopt(argc,argv,"n:h:i:o:vd"); - if (c==-1) break; - switch (c) { - case 'v': - verbose=1; - break; - case 'n': - nfft=atoi(optarg); - break; - case 'i': - fin = fopen(optarg,"rb"); - if (fin==NULL) { - perror(optarg); - exit(1); - } - break; - case 'o': - fout = fopen(optarg,"w+b"); - if (fout==NULL) { - perror(optarg); - exit(1); - } - break; - case 'h': - filtfile = fopen(optarg,"rb"); - if (filtfile==NULL) { - perror(optarg); - exit(1); - } - break; - case 'd': - use_direct=1; - break; - case '?': - fprintf(stderr,"usage options:\n" - "\t-n nfft: fft size to use\n" - "\t-d : use direct FIR filtering, not fast convolution\n" - "\t-i filename: input file\n" - "\t-o filename: output(filtered) file\n" - "\t-n nfft: fft size to use\n" - "\t-h filename: impulse response\n"); - exit (1); - default:fprintf(stderr,"bad %c\n",c);break; - } - } - if (filtfile==NULL) { - fprintf(stderr,"You must supply the FIR coeffs via -h\n"); - exit(1); - } - fseek(filtfile,0,SEEK_END); - nh = ftell(filtfile) / sizeof(kffsamp_t); - if (verbose) fprintf(stderr,"%d samples in FIR filter\n",(int)nh); - h = (kffsamp_t*)malloc(sizeof(kffsamp_t)*nh); - fseek(filtfile,0,SEEK_SET); - if (fread(h,sizeof(kffsamp_t),nh,filtfile) != nh) - fprintf(stderr,"short read on filter file\n"); - - fclose(filtfile); - - if (use_direct) - direct_file_filter( fin, fout, h,nh); - else - do_file_filter( fin, fout, h,nh,nfft); - - if (fout!=stdout) fclose(fout); - if (fin!=stdin) fclose(fin); - - return 0; -} -#endif diff --git a/native_client/kiss_fft130/tools/kiss_fftnd.c b/native_client/kiss_fft130/tools/kiss_fftnd.c deleted file mode 100644 index d6c91243a8..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftnd.c +++ /dev/null @@ -1,193 +0,0 @@ - - -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "kiss_fftnd.h" -#include "_kiss_fft_guts.h" - -struct kiss_fftnd_state{ - int dimprod; /* dimsum would be mighty tasty right now */ - int ndims; - int *dims; - kiss_fft_cfg *states; /* cfg states for each dimension */ - kiss_fft_cpx * tmpbuf; /*buffer capable of hold the entire input */ -}; - -kiss_fftnd_cfg kiss_fftnd_alloc(const int *dims,int ndims,int inverse_fft,void*mem,size_t*lenmem) -{ - kiss_fftnd_cfg st = NULL; - int i; - int dimprod=1; - size_t memneeded = sizeof(struct kiss_fftnd_state); - char * ptr; - - for (i=0;istates[i] */ - dimprod *= dims[i]; - } - memneeded += sizeof(int) * ndims;/* st->dims */ - memneeded += sizeof(void*) * ndims;/* st->states */ - memneeded += sizeof(kiss_fft_cpx) * dimprod; /* st->tmpbuf */ - - if (lenmem == NULL) {/* allocate for the caller*/ - st = (kiss_fftnd_cfg) malloc (memneeded); - } else { /* initialize supplied buffer if big enough */ - if (*lenmem >= memneeded) - st = (kiss_fftnd_cfg) mem; - *lenmem = memneeded; /*tell caller how big struct is (or would be) */ - } - if (!st) - return NULL; /*malloc failed or buffer too small */ - - st->dimprod = dimprod; - st->ndims = ndims; - ptr=(char*)(st+1); - - st->states = (kiss_fft_cfg *)ptr; - ptr += sizeof(void*) * ndims; - - st->dims = (int*)ptr; - ptr += sizeof(int) * ndims; - - st->tmpbuf = (kiss_fft_cpx*)ptr; - ptr += sizeof(kiss_fft_cpx) * dimprod; - - for (i=0;idims[i] = dims[i]; - kiss_fft_alloc (st->dims[i], inverse_fft, NULL, &len); - st->states[i] = kiss_fft_alloc (st->dims[i], inverse_fft, ptr,&len); - ptr += len; - } - /* -Hi there! - -If you're looking at this particular code, it probably means you've got a brain-dead bounds checker -that thinks the above code overwrites the end of the array. - -It doesn't. - --- Mark - -P.S. -The below code might give you some warm fuzzies and help convince you. - */ - if ( ptr - (char*)st != (int)memneeded ) { - fprintf(stderr, - "################################################################################\n" - "Internal error! Memory allocation miscalculation\n" - "################################################################################\n" - ); - } - return st; -} - -/* - This works by tackling one dimension at a time. - - In effect, - Each stage starts out by reshaping the matrix into a DixSi 2d matrix. - A Di-sized fft is taken of each column, transposing the matrix as it goes. - -Here's a 3-d example: -Take a 2x3x4 matrix, laid out in memory as a contiguous buffer - [ [ [ a b c d ] [ e f g h ] [ i j k l ] ] - [ [ m n o p ] [ q r s t ] [ u v w x ] ] ] - -Stage 0 ( D=2): treat the buffer as a 2x12 matrix - [ [a b ... k l] - [m n ... w x] ] - - FFT each column with size 2. - Transpose the matrix at the same time using kiss_fft_stride. - - [ [ a+m a-m ] - [ b+n b-n] - ... - [ k+w k-w ] - [ l+x l-x ] ] - - Note fft([x y]) == [x+y x-y] - -Stage 1 ( D=3) treats the buffer (the output of stage D=2) as an 3x8 matrix, - [ [ a+m a-m b+n b-n c+o c-o d+p d-p ] - [ e+q e-q f+r f-r g+s g-s h+t h-t ] - [ i+u i-u j+v j-v k+w k-w l+x l-x ] ] - - And perform FFTs (size=3) on each of the columns as above, transposing - the matrix as it goes. The output of stage 1 is - (Legend: ap = [ a+m e+q i+u ] - am = [ a-m e-q i-u ] ) - - [ [ sum(ap) fft(ap)[0] fft(ap)[1] ] - [ sum(am) fft(am)[0] fft(am)[1] ] - [ sum(bp) fft(bp)[0] fft(bp)[1] ] - [ sum(bm) fft(bm)[0] fft(bm)[1] ] - [ sum(cp) fft(cp)[0] fft(cp)[1] ] - [ sum(cm) fft(cm)[0] fft(cm)[1] ] - [ sum(dp) fft(dp)[0] fft(dp)[1] ] - [ sum(dm) fft(dm)[0] fft(dm)[1] ] ] - -Stage 2 ( D=4) treats this buffer as a 4*6 matrix, - [ [ sum(ap) fft(ap)[0] fft(ap)[1] sum(am) fft(am)[0] fft(am)[1] ] - [ sum(bp) fft(bp)[0] fft(bp)[1] sum(bm) fft(bm)[0] fft(bm)[1] ] - [ sum(cp) fft(cp)[0] fft(cp)[1] sum(cm) fft(cm)[0] fft(cm)[1] ] - [ sum(dp) fft(dp)[0] fft(dp)[1] sum(dm) fft(dm)[0] fft(dm)[1] ] ] - - Then FFTs each column, transposing as it goes. - - The resulting matrix is the 3d FFT of the 2x3x4 input matrix. - - Note as a sanity check that the first element of the final - stage's output (DC term) is - sum( [ sum(ap) sum(bp) sum(cp) sum(dp) ] ) - , i.e. the summation of all 24 input elements. - -*/ -void kiss_fftnd(kiss_fftnd_cfg st,const kiss_fft_cpx *fin,kiss_fft_cpx *fout) -{ - int i,k; - const kiss_fft_cpx * bufin=fin; - kiss_fft_cpx * bufout; - - /*arrange it so the last bufout == fout*/ - if ( st->ndims & 1 ) { - bufout = fout; - if (fin==fout) { - memcpy( st->tmpbuf, fin, sizeof(kiss_fft_cpx) * st->dimprod ); - bufin = st->tmpbuf; - } - }else - bufout = st->tmpbuf; - - for ( k=0; k < st->ndims; ++k) { - int curdim = st->dims[k]; - int stride = st->dimprod / curdim; - - for ( i=0 ; istates[k], bufin+i , bufout+i*curdim, stride ); - - /*toggle back and forth between the two buffers*/ - if (bufout == st->tmpbuf){ - bufout = fout; - bufin = st->tmpbuf; - }else{ - bufout = st->tmpbuf; - bufin = fout; - } - } -} diff --git a/native_client/kiss_fft130/tools/kiss_fftnd.h b/native_client/kiss_fft130/tools/kiss_fftnd.h deleted file mode 100644 index 42e7df5b54..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftnd.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef KISS_FFTND_H -#define KISS_FFTND_H - -#include "kiss_fft.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct kiss_fftnd_state * kiss_fftnd_cfg; - -kiss_fftnd_cfg kiss_fftnd_alloc(const int *dims,int ndims,int inverse_fft,void*mem,size_t*lenmem); -void kiss_fftnd(kiss_fftnd_cfg cfg,const kiss_fft_cpx *fin,kiss_fft_cpx *fout); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/native_client/kiss_fft130/tools/kiss_fftndr.c b/native_client/kiss_fft130/tools/kiss_fftndr.c deleted file mode 100644 index ba550dd10b..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftndr.c +++ /dev/null @@ -1,118 +0,0 @@ -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "kiss_fftndr.h" -#include "_kiss_fft_guts.h" -#define MAX(x,y) ( ( (x)<(y) )?(y):(x) ) - -struct kiss_fftndr_state -{ - int dimReal; - int dimOther; - kiss_fftr_cfg cfg_r; - kiss_fftnd_cfg cfg_nd; - void * tmpbuf; -}; - -static int prod(const int *dims, int ndims) -{ - int x=1; - while (ndims--) - x *= *dims++; - return x; -} - -kiss_fftndr_cfg kiss_fftndr_alloc(const int *dims,int ndims,int inverse_fft,void*mem,size_t*lenmem) -{ - kiss_fftndr_cfg st = NULL; - size_t nr=0 , nd=0,ntmp=0; - int dimReal = dims[ndims-1]; - int dimOther = prod(dims,ndims-1); - size_t memneeded; - - (void)kiss_fftr_alloc(dimReal,inverse_fft,NULL,&nr); - (void)kiss_fftnd_alloc(dims,ndims-1,inverse_fft,NULL,&nd); - ntmp = - MAX( 2*dimOther , dimReal+2) * sizeof(kiss_fft_scalar) // freq buffer for one pass - + dimOther*(dimReal+2) * sizeof(kiss_fft_scalar); // large enough to hold entire input in case of in-place - - memneeded = sizeof( struct kiss_fftndr_state ) + nr + nd + ntmp; - - if (lenmem==NULL) { - st = (kiss_fftndr_cfg) malloc(memneeded); - }else{ - if (*lenmem >= memneeded) - st = (kiss_fftndr_cfg)mem; - *lenmem = memneeded; - } - if (st==NULL) - return NULL; - memset( st , 0 , memneeded); - - st->dimReal = dimReal; - st->dimOther = dimOther; - st->cfg_r = kiss_fftr_alloc( dimReal,inverse_fft,st+1,&nr); - st->cfg_nd = kiss_fftnd_alloc(dims,ndims-1,inverse_fft, ((char*) st->cfg_r)+nr,&nd); - st->tmpbuf = (char*)st->cfg_nd + nd; - - return st; -} - -void kiss_fftndr(kiss_fftndr_cfg st,const kiss_fft_scalar *timedata,kiss_fft_cpx *freqdata) -{ - int k1,k2; - int dimReal = st->dimReal; - int dimOther = st->dimOther; - int nrbins = dimReal/2+1; - - kiss_fft_cpx * tmp1 = (kiss_fft_cpx*)st->tmpbuf; - kiss_fft_cpx * tmp2 = tmp1 + MAX(nrbins,dimOther); - - // timedata is N0 x N1 x ... x Nk real - - // take a real chunk of data, fft it and place the output at correct intervals - for (k1=0;k1cfg_r, timedata + k1*dimReal , tmp1 ); // tmp1 now holds nrbins complex points - for (k2=0;k2cfg_nd, tmp2+k2*dimOther, tmp1); // tmp1 now holds dimOther complex points - for (k1=0;k1dimReal; - int dimOther = st->dimOther; - int nrbins = dimReal/2+1; - kiss_fft_cpx * tmp1 = (kiss_fft_cpx*)st->tmpbuf; - kiss_fft_cpx * tmp2 = tmp1 + MAX(nrbins,dimOther); - - for (k2=0;k2cfg_nd, tmp1, tmp2+k2*dimOther); - } - - for (k1=0;k1cfg_r,tmp1,timedata + k1*dimReal); - } -} diff --git a/native_client/kiss_fft130/tools/kiss_fftndr.h b/native_client/kiss_fft130/tools/kiss_fftndr.h deleted file mode 100644 index 38ec3ab023..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftndr.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef KISS_NDR_H -#define KISS_NDR_H - -#include "kiss_fft.h" -#include "kiss_fftr.h" -#include "kiss_fftnd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct kiss_fftndr_state *kiss_fftndr_cfg; - - -kiss_fftndr_cfg kiss_fftndr_alloc(const int *dims,int ndims,int inverse_fft,void*mem,size_t*lenmem); -/* - dims[0] must be even - - If you don't care to allocate space, use mem = lenmem = NULL -*/ - - -void kiss_fftndr( - kiss_fftndr_cfg cfg, - const kiss_fft_scalar *timedata, - kiss_fft_cpx *freqdata); -/* - input timedata has dims[0] X dims[1] X ... X dims[ndims-1] scalar points - output freqdata has dims[0] X dims[1] X ... X dims[ndims-1]/2+1 complex points -*/ - -void kiss_fftndri( - kiss_fftndr_cfg cfg, - const kiss_fft_cpx *freqdata, - kiss_fft_scalar *timedata); -/* - input and output dimensions are the exact opposite of kiss_fftndr -*/ - - -#define kiss_fftr_free free - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/native_client/kiss_fft130/tools/kiss_fftr.c b/native_client/kiss_fft130/tools/kiss_fftr.c deleted file mode 100644 index b8e238b1e2..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftr.c +++ /dev/null @@ -1,159 +0,0 @@ -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include "kiss_fftr.h" -#include "_kiss_fft_guts.h" - -struct kiss_fftr_state{ - kiss_fft_cfg substate; - kiss_fft_cpx * tmpbuf; - kiss_fft_cpx * super_twiddles; -#ifdef USE_SIMD - void * pad; -#endif -}; - -kiss_fftr_cfg kiss_fftr_alloc(int nfft,int inverse_fft,void * mem,size_t * lenmem) -{ - int i; - kiss_fftr_cfg st = NULL; - size_t subsize, memneeded; - - if (nfft & 1) { - fprintf(stderr,"Real FFT optimization must be even.\n"); - return NULL; - } - nfft >>= 1; - - kiss_fft_alloc (nfft, inverse_fft, NULL, &subsize); - memneeded = sizeof(struct kiss_fftr_state) + subsize + sizeof(kiss_fft_cpx) * ( nfft * 3 / 2); - - if (lenmem == NULL) { - st = (kiss_fftr_cfg) KISS_FFT_MALLOC (memneeded); - } else { - if (*lenmem >= memneeded) - st = (kiss_fftr_cfg) mem; - *lenmem = memneeded; - } - if (!st) - return NULL; - - st->substate = (kiss_fft_cfg) (st + 1); /*just beyond kiss_fftr_state struct */ - st->tmpbuf = (kiss_fft_cpx *) (((char *) st->substate) + subsize); - st->super_twiddles = st->tmpbuf + nfft; - kiss_fft_alloc(nfft, inverse_fft, st->substate, &subsize); - - for (i = 0; i < nfft/2; ++i) { - double phase = - -3.14159265358979323846264338327 * ((double) (i+1) / nfft + .5); - if (inverse_fft) - phase *= -1; - kf_cexp (st->super_twiddles+i,phase); - } - return st; -} - -void kiss_fftr(kiss_fftr_cfg st,const kiss_fft_scalar *timedata,kiss_fft_cpx *freqdata) -{ - /* input buffer timedata is stored row-wise */ - int k,ncfft; - kiss_fft_cpx fpnk,fpk,f1k,f2k,tw,tdc; - - if ( st->substate->inverse) { - fprintf(stderr,"kiss fft usage error: improper alloc\n"); - exit(1); - } - - ncfft = st->substate->nfft; - - /*perform the parallel fft of two real signals packed in real,imag*/ - kiss_fft( st->substate , (const kiss_fft_cpx*)timedata, st->tmpbuf ); - /* The real part of the DC element of the frequency spectrum in st->tmpbuf - * contains the sum of the even-numbered elements of the input time sequence - * The imag part is the sum of the odd-numbered elements - * - * The sum of tdc.r and tdc.i is the sum of the input time sequence. - * yielding DC of input time sequence - * The difference of tdc.r - tdc.i is the sum of the input (dot product) [1,-1,1,-1... - * yielding Nyquist bin of input time sequence - */ - - tdc.r = st->tmpbuf[0].r; - tdc.i = st->tmpbuf[0].i; - C_FIXDIV(tdc,2); - CHECK_OVERFLOW_OP(tdc.r ,+, tdc.i); - CHECK_OVERFLOW_OP(tdc.r ,-, tdc.i); - freqdata[0].r = tdc.r + tdc.i; - freqdata[ncfft].r = tdc.r - tdc.i; -#ifdef USE_SIMD - freqdata[ncfft].i = freqdata[0].i = _mm_set1_ps(0); -#else - freqdata[ncfft].i = freqdata[0].i = 0; -#endif - - for ( k=1;k <= ncfft/2 ; ++k ) { - fpk = st->tmpbuf[k]; - fpnk.r = st->tmpbuf[ncfft-k].r; - fpnk.i = - st->tmpbuf[ncfft-k].i; - C_FIXDIV(fpk,2); - C_FIXDIV(fpnk,2); - - C_ADD( f1k, fpk , fpnk ); - C_SUB( f2k, fpk , fpnk ); - C_MUL( tw , f2k , st->super_twiddles[k-1]); - - freqdata[k].r = HALF_OF(f1k.r + tw.r); - freqdata[k].i = HALF_OF(f1k.i + tw.i); - freqdata[ncfft-k].r = HALF_OF(f1k.r - tw.r); - freqdata[ncfft-k].i = HALF_OF(tw.i - f1k.i); - } -} - -void kiss_fftri(kiss_fftr_cfg st,const kiss_fft_cpx *freqdata,kiss_fft_scalar *timedata) -{ - /* input buffer timedata is stored row-wise */ - int k, ncfft; - - if (st->substate->inverse == 0) { - fprintf (stderr, "kiss fft usage error: improper alloc\n"); - exit (1); - } - - ncfft = st->substate->nfft; - - st->tmpbuf[0].r = freqdata[0].r + freqdata[ncfft].r; - st->tmpbuf[0].i = freqdata[0].r - freqdata[ncfft].r; - C_FIXDIV(st->tmpbuf[0],2); - - for (k = 1; k <= ncfft / 2; ++k) { - kiss_fft_cpx fk, fnkc, fek, fok, tmp; - fk = freqdata[k]; - fnkc.r = freqdata[ncfft - k].r; - fnkc.i = -freqdata[ncfft - k].i; - C_FIXDIV( fk , 2 ); - C_FIXDIV( fnkc , 2 ); - - C_ADD (fek, fk, fnkc); - C_SUB (tmp, fk, fnkc); - C_MUL (fok, tmp, st->super_twiddles[k-1]); - C_ADD (st->tmpbuf[k], fek, fok); - C_SUB (st->tmpbuf[ncfft - k], fek, fok); -#ifdef USE_SIMD - st->tmpbuf[ncfft - k].i *= _mm_set1_ps(-1.0); -#else - st->tmpbuf[ncfft - k].i *= -1; -#endif - } - kiss_fft (st->substate, st->tmpbuf, (kiss_fft_cpx *) timedata); -} diff --git a/native_client/kiss_fft130/tools/kiss_fftr.h b/native_client/kiss_fft130/tools/kiss_fftr.h deleted file mode 100644 index 72e5a57714..0000000000 --- a/native_client/kiss_fft130/tools/kiss_fftr.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef KISS_FTR_H -#define KISS_FTR_H - -#include "kiss_fft.h" -#ifdef __cplusplus -extern "C" { -#endif - - -/* - - Real optimized version can save about 45% cpu time vs. complex fft of a real seq. - - - - */ - -typedef struct kiss_fftr_state *kiss_fftr_cfg; - - -kiss_fftr_cfg kiss_fftr_alloc(int nfft,int inverse_fft,void * mem, size_t * lenmem); -/* - nfft must be even - - If you don't care to allocate space, use mem = lenmem = NULL -*/ - - -void kiss_fftr(kiss_fftr_cfg cfg,const kiss_fft_scalar *timedata,kiss_fft_cpx *freqdata); -/* - input timedata has nfft scalar points - output freqdata has nfft/2+1 complex points -*/ - -void kiss_fftri(kiss_fftr_cfg cfg,const kiss_fft_cpx *freqdata,kiss_fft_scalar *timedata); -/* - input freqdata has nfft/2+1 complex points - output timedata has nfft scalar points -*/ - -#define kiss_fftr_free free - -#ifdef __cplusplus -} -#endif -#endif diff --git a/native_client/kiss_fft130/tools/psdpng.c b/native_client/kiss_fft130/tools/psdpng.c deleted file mode 100644 index d11a54fd2e..0000000000 --- a/native_client/kiss_fft130/tools/psdpng.c +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright (c) 2003-2004, Mark Borgerding - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the author nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#include -#include -#include -#include -#include -#include - -#include "kiss_fft.h" -#include "kiss_fftr.h" - -int nfft=1024; -FILE * fin=NULL; -FILE * fout=NULL; - -int navg=20; -int remove_dc=0; -int nrows=0; -float * vals=NULL; -int stereo=0; - -static -void config(int argc,char** argv) -{ - while (1) { - int c = getopt (argc, argv, "n:r:as"); - if (c == -1) - break; - switch (c) { - case 'n': nfft=(int)atoi(optarg);break; - case 'r': navg=(int)atoi(optarg);break; - case 'a': remove_dc=1;break; - case 's': stereo=1;break; - case '?': - fprintf (stderr, "usage options:\n" - "\t-n d: fft dimension(s) [1024]\n" - "\t-r d: number of rows to average [20]\n" - "\t-a : remove average from each fft buffer\n" - "\t-s : input is stereo, channels will be combined before fft\n" - "16 bit machine format real input is assumed\n" - ); - default: - fprintf (stderr, "bad %c\n", c); - exit (1); - break; - } - } - if ( optind < argc ) { - if (strcmp("-",argv[optind]) !=0) - fin = fopen(argv[optind],"rb"); - ++optind; - } - - if ( optind < argc ) { - if ( strcmp("-",argv[optind]) !=0 ) - fout = fopen(argv[optind],"wb"); - ++optind; - } - if (fin==NULL) - fin=stdin; - if (fout==NULL) - fout=stdout; -} - -#define CHECKNULL(p) if ( (p)==NULL ) do { fprintf(stderr,"CHECKNULL failed @ %s(%d): %s\n",__FILE__,__LINE__,#p );exit(1);} while(0) - -typedef struct -{ - png_byte r; - png_byte g; - png_byte b; -} rgb_t; - -static -void val2rgb(float x,rgb_t *p) -{ - const double pi = 3.14159265358979; - p->g = (int)(255*sin(x*pi)); - p->r = (int)(255*abs(sin(x*pi*3/2))); - p->b = (int)(255*abs(sin(x*pi*5/2))); - //fprintf(stderr,"%.2f : %d,%d,%d\n",x,(int)p->r,(int)p->g,(int)p->b); -} - -static -void cpx2pixels(rgb_t * res,const float * fbuf,size_t n) -{ - size_t i; - float minval,maxval,valrange; - minval=maxval=fbuf[0]; - - for (i = 0; i < n; ++i) { - if (fbuf[i] > maxval) maxval = fbuf[i]; - if (fbuf[i] < minval) minval = fbuf[i]; - } - - fprintf(stderr,"min ==%f,max=%f\n",minval,maxval); - valrange = maxval-minval; - if (valrange == 0) { - fprintf(stderr,"min == max == %f\n",minval); - exit (1); - } - - for (i = 0; i < n; ++i) - val2rgb( (fbuf[i] - minval)/valrange , res+i ); -} - -static -void transform_signal(void) -{ - short *inbuf; - kiss_fftr_cfg cfg=NULL; - kiss_fft_scalar *tbuf; - kiss_fft_cpx *fbuf; - float *mag2buf; - int i; - int n; - int avgctr=0; - - int nfreqs=nfft/2+1; - - CHECKNULL( cfg=kiss_fftr_alloc(nfft,0,0,0) ); - CHECKNULL( inbuf=(short*)malloc(sizeof(short)*2*nfft ) ); - CHECKNULL( tbuf=(kiss_fft_scalar*)malloc(sizeof(kiss_fft_scalar)*nfft ) ); - CHECKNULL( fbuf=(kiss_fft_cpx*)malloc(sizeof(kiss_fft_cpx)*nfreqs ) ); - CHECKNULL( mag2buf=(float*)malloc(sizeof(float)*nfreqs ) ); - - memset(mag2buf,0,sizeof(mag2buf)*nfreqs); - - while (1) { - if (stereo) { - n = fread(inbuf,sizeof(short)*2,nfft,fin); - if (n != nfft ) - break; - for (i=0;i= 2.1 -python_speech_features -pyxdg -bs4 +progressbar2 +pandas six -requests -tables +pyxdg attrdict + +# Requirements for building native_client files setuptools + +# Requirements for importers +sox +bs4 +requests librosa soundfile + +# Miscellaneous scripts +paramiko >= 2.1 +scipy +matplotlib diff --git a/tc-single-shot-inference.sh b/tc-single-shot-inference.sh index e7d229d722..7f3f9fe566 100755 --- a/tc-single-shot-inference.sh +++ b/tc-single-shot-inference.sh @@ -33,6 +33,7 @@ export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH" mkdir -p ${PYENV_ROOT} || true mkdir -p ${TASKCLUSTER_ARTIFACTS} || true mkdir -p /tmp/train || true +mkdir -p /tmp/train_tflite || true install_pyenv "${PYENV_ROOT}" install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index ea698b114a..c114fb6fba 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -67,6 +67,22 @@ assert_correct_inference() { phrase=$(strip "$1") expected=$(strip "$2") + status=$3 + + if [ "$status" -ne "0" ]; then + case "$(cat ${TASKCLUSTER_TMP_DIR}/stderr)" in + *"incompatible with minimum version"*) + echo "Prod model too old for client, skipping test." + return 0 + ;; + + *) + echo "Client failed to run:" + cat ${TASKCLUSTER_TMP_DIR}/stderr + return 1 + ;; + esac + fi if [ -z "${phrase}" -o -z "${expected}" ]; then echo "One or more empty strings:" @@ -95,6 +111,7 @@ assert_working_inference() { phrase=$1 expected=$2 + status=$3 if [ -z "${phrase}" -o -z "${expected}" ]; then echo "One or more empty strings:" @@ -103,6 +120,21 @@ assert_working_inference() return 1 fi; + if [ "$status" -ne "0" ]; then + case "$(cat ${TASKCLUSTER_TMP_DIR}/stderr)" in + *"incompatible with minimum version"*) + echo "Prod model too old for client, skipping test." + return 0 + ;; + + *) + echo "Client failed to run:" + cat ${TASKCLUSTER_TMP_DIR}/stderr + return 1 + ;; + esac + fi + case "${phrase}" in *${expected}*) echo "Proper output has been produced:" @@ -135,6 +167,11 @@ assert_shows_something() fi; case "${stderr}" in + *"incompatible with minimum version"*) + echo "Prod model too old for client, skipping test." + return 0 + ;; + *${expected}*) echo "Proper output has been produced:" echo "${stderr}" @@ -186,40 +223,40 @@ assert_not_present() assert_correct_ldc93s1() { - assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" "$2" } assert_working_ldc93s1() { - assert_working_inference "$1" "she had your dark suit in greasy wash water all year" + assert_working_inference "$1" "she had your dark suit in greasy wash water all year" "$2" } assert_correct_ldc93s1_lm() { - assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" "$2" } assert_working_ldc93s1_lm() { - assert_working_inference "$1" "she had your dark suit in greasy wash water all year" + assert_working_inference "$1" "she had your dark suit in greasy wash water all year" "$2" } assert_correct_multi_ldc93s1() { - assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%" - assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%" + assert_shows_something "$1" "/LDC93S1.wav%she had your dark suit in greasy wash water all year%" "$?" + assert_shows_something "$1" "/LDC93S1_pcms16le_2_44100.wav%she had your dark suit in greasy wash water all year%" "$?" ## 8k will output garbage anyway ... # assert_shows_something "$1" "/LDC93S1_pcms16le_1_8000.wav%she hayorasryrtl lyreasy asr watal w water all year%" } assert_correct_ldc93s1_prodmodel() { - assert_correct_inference "$1" "she had a due and greasy wash water year" + assert_correct_inference "$1" "she had a due and greasy wash water year" "$2" } assert_correct_ldc93s1_prodmodel_stereo_44k() { - assert_correct_inference "$1" "she had a due and greasy wash water year" + assert_correct_inference "$1" "she had a due and greasy wash water year" "$2" } assert_correct_warning_upsampling() @@ -249,73 +286,117 @@ check_tensorflow_version() run_tflite_basic_inference_tests() { - phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}deepspeech --model ${ANDROID_TMP_DIR}/ds/${model_name} --alphabet ${ANDROID_TMP_DIR}/ds/alphabet.txt --audio ${ANDROID_TMP_DIR}/ds/LDC93S1.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" + set +e + phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}deepspeech --model ${ANDROID_TMP_DIR}/ds/${model_name} --alphabet ${ANDROID_TMP_DIR}/ds/alphabet.txt --audio ${ANDROID_TMP_DIR}/ds/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + set -e + assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$?" } run_netframework_inference_tests() { - phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_working_ldc93s1 "${phrase_pbmodel_nolm}" + set +e + phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + set -e + assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" - phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_working_ldc93s1 "${phrase_pbmodel_nolm}" + set +e + phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + set -e + assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" - phrase_pbmodel_withlm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" + set +e + phrase_pbmodel_withlm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + set -e + assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" "$?" } run_basic_inference_tests() { - phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" + set +e + phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status" - phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" + set +e + phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status" - phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" + set +e + phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status" } run_all_inference_tests() { run_basic_inference_tests - phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) - assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" + set +e + phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" "$status" - phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) - assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}" + set +e + phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}" "$status" + set +e phrase_pbmodel_nolm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + set -e assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}" + set +e phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + set -e assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" } run_prod_inference_tests() { - phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" + set +e + phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" - phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav) - assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" + set +e + phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" - phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav) - assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" + set +e + phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + status=$? + set -e + assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" + set +e phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + set -e assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" } run_multi_inference_tests() { - multi_phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/ | tr '\n' '%') - assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_nolm}" - - multi_phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/ | tr '\n' '%') - assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_withlm}" + set +e -o pipefail + multi_phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%') + status=$? + set -e +o pipefail + assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_nolm}" "$status" + + set +e -o pipefail + multi_phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --alphabet ${TASKCLUSTER_TMP_DIR}/alphabet.txt --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%') + status=$? + set -e +o pipefail + assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_withlm}" "$status" } android_run_tests() diff --git a/tc-train-tests.sh b/tc-train-tests.sh index bc6d5a172d..a08624b4cd 100644 --- a/tc-train-tests.sh +++ b/tc-train-tests.sh @@ -32,6 +32,7 @@ export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH" mkdir -p ${PYENV_ROOT} || true mkdir -p ${TASKCLUSTER_ARTIFACTS} || true mkdir -p /tmp/train || true +mkdir -p /tmp/train_tflite || true install_pyenv "${PYENV_ROOT}" install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" @@ -57,13 +58,13 @@ LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-bin pushd ${HOME}/DeepSpeech/ds/ # Run twice to test preprocessed features - time ./bin/run-tc-ldc93s1_new.sh 104 - time ./bin/run-tc-ldc93s1_new.sh 105 + time ./bin/run-tc-ldc93s1_new.sh 199 + time ./bin/run-tc-ldc93s1_new.sh 200 time ./bin/run-tc-ldc93s1_tflite.sh popd cp /tmp/train/output_graph.pb ${TASKCLUSTER_ARTIFACTS} -cp /tmp/train/output_graph.tflite ${TASKCLUSTER_ARTIFACTS} +cp /tmp/train_tflite/output_graph.tflite ${TASKCLUSTER_ARTIFACTS} if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then convert_graphdef=$(basename "${CONVERT_GRAPHDEF_MEMMAPPED}") @@ -74,7 +75,7 @@ if [ ! -z "${CONVERT_GRAPHDEF_MEMMAPPED}" ]; then fi; pushd ${HOME}/DeepSpeech/ds/ - time ./bin/run-tc-ldc93s1_checkpoint.sh 105 + time ./bin/run-tc-ldc93s1_checkpoint.sh 200 popd deactivate diff --git a/util/audio.py b/util/audio.py deleted file mode 100644 index ac9dde6390..0000000000 --- a/util/audio.py +++ /dev/null @@ -1,24 +0,0 @@ -import numpy as np -import scipy.io.wavfile as wav - -from python_speech_features import mfcc - - -def audiofile_to_input_vector(audio_filename, numcep, numcontext): - r""" - Given a WAV audio file at ``audio_filename``, calculates ``numcep`` MFCC features - at every 0.01s time step with a window length of 0.025s. Appends ``numcontext`` - context frames to the left and right of each time step, and returns this data - in a numpy array. - """ - # Load wav files - fs, audio = wav.read(audio_filename) - - # Get mfcc coefficients - features = mfcc(audio, samplerate=fs, numcep=numcep, winlen=0.032, winstep=0.02, winfunc=np.hamming) - - # Add empty initial and final contexts - empty_context = np.zeros((numcontext, numcep), dtype=features.dtype) - features = np.concatenate((empty_context, features, empty_context)) - - return features diff --git a/util/config.py b/util/config.py index 378bb8f2ca..b776545198 100644 --- a/util/config.py +++ b/util/config.py @@ -92,10 +92,7 @@ def initialize_globals(): # Units in the sixth layer = number of characters in the target language plus one c.n_hidden_6 = c.alphabet.size() + 1 # +1 for CTC blank label - if len(FLAGS.one_shot_infer) > 0: - FLAGS.train = False - FLAGS.test = False - FLAGS.export_dir = '' + if FLAGS.one_shot_infer: if not os.path.exists(FLAGS.one_shot_infer): log_error('Path specified in --one_shot_infer is not a valid file.') exit(1) diff --git a/util/feeding.py b/util/feeding.py index 71e2a9fcec..2feb5bbc40 100644 --- a/util/feeding.py +++ b/util/feeding.py @@ -1,198 +1,97 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function + import numpy as np +import os +import pandas import tensorflow as tf -from math import ceil -from six.moves import range -from threading import Thread -from util.gpu import get_available_gpus - - -class ModelFeeder(object): - ''' - Feeds data into a model. - Feeding is parallelized by independent units called tower feeders (usually one per GPU). - Each tower feeder provides data from runtime switchable sources (train, dev). - These sources are to be provided by the DataSet instances whose references are kept. - Creates, owns and delegates to tower_feeder_count internal tower feeder objects. - ''' - def __init__(self, - train_set, - dev_set, - numcep, - numcontext, - alphabet, - tower_feeder_count=-1, - threads_per_queue=4): - - self.train = train_set - self.dev = dev_set - self.sets = [train_set, dev_set] - self.numcep = numcep - self.numcontext = numcontext - self.tower_feeder_count = max(len(get_available_gpus()), 1) if tower_feeder_count < 0 else tower_feeder_count - self.threads_per_queue = threads_per_queue - - self.ph_x = tf.placeholder(tf.float32, [None, 2*numcontext+1, numcep]) - self.ph_x_length = tf.placeholder(tf.int32, []) - self.ph_y = tf.placeholder(tf.int32, [None,]) - self.ph_y_length = tf.placeholder(tf.int32, []) - self.ph_batch_size = tf.placeholder(tf.int32, []) - self.ph_queue_selector = tf.placeholder(tf.int32, name='Queue_Selector') - - self._tower_feeders = [_TowerFeeder(self, i, alphabet) for i in range(self.tower_feeder_count)] - - def start_queue_threads(self, session, coord): - ''' - Starts required queue threads on all tower feeders. - ''' - queue_threads = [] - for tower_feeder in self._tower_feeders: - queue_threads += tower_feeder.start_queue_threads(session, coord) - return queue_threads - - def close_queues(self, session): - ''' - Closes queues of all tower feeders. - ''' - for tower_feeder in self._tower_feeders: - tower_feeder.close_queues(session) - - def set_data_set(self, feed_dict, data_set): - ''' - Switches all tower feeders to a different source DataSet. - The provided feed_dict will get enriched with required placeholder/value pairs. - The DataSet has to be one of those that got passed into the constructor. - ''' - index = self.sets.index(data_set) - assert index >= 0 - feed_dict[self.ph_queue_selector] = index - feed_dict[self.ph_batch_size] = data_set.batch_size - - def next_batch(self, tower_feeder_index): - ''' - Draw the next batch from one of the tower feeders. - ''' - return self._tower_feeders[tower_feeder_index].next_batch() - - -class DataSet(object): - ''' - Represents a collection of audio samples and their respective transcriptions. - Takes a set of CSV files produced by importers in /bin. - ''' - def __init__(self, data, batch_size, skip=0, limit=0, ascending=True, next_index=lambda i: i + 1): - self.data = data - self.data.sort_values(by="features_len", ascending=ascending, inplace=True) - self.batch_size = batch_size - self.next_index = next_index - self.total_batches = int(ceil(len(self.data) / batch_size)) - - -class _DataSetLoader(object): - ''' - Internal class that represents an input queue with data from one of the DataSet objects. - Each tower feeder will create and combine three data set loaders to one switchable queue. - Keeps a ModelFeeder reference for accessing shared settings and placeholders. - Keeps a DataSet reference to access its samples. - ''' - def __init__(self, model_feeder, data_set, alphabet): - self._model_feeder = model_feeder - self._data_set = data_set - self.queue = tf.PaddingFIFOQueue(shapes=[[None, 2 * model_feeder.numcontext + 1, model_feeder.numcep], [], [None,], []], - dtypes=[tf.float32, tf.int32, tf.int32, tf.int32], - capacity=data_set.batch_size * 8) - self._enqueue_op = self.queue.enqueue([model_feeder.ph_x, model_feeder.ph_x_length, model_feeder.ph_y, model_feeder.ph_y_length]) - self._close_op = self.queue.close(cancel_pending_enqueues=True) - self._alphabet = alphabet - - def start_queue_threads(self, session, coord): - ''' - Starts concurrent queue threads for reading samples from the data set. - ''' - queue_threads = [Thread(target=self._populate_batch_queue, args=(session, coord)) - for i in range(self._model_feeder.threads_per_queue)] - for queue_thread in queue_threads: - coord.register_thread(queue_thread) - queue_thread.daemon = True - queue_thread.start() - return queue_threads - - def close_queue(self, session): - ''' - Closes the data set queue. - ''' - session.run(self._close_op) - - def _populate_batch_queue(self, session, coord): - ''' - Queue thread routine. - ''' - file_count = len(self._data_set.data) - index = -1 - while not coord.should_stop(): - index = self._data_set.next_index(index) % file_count - features, num_strides, transcript, transcript_len = self._data_set.data.iloc[index] - - # Create a view into the array with overlapping strides of size - # numcontext (past) + 1 (present) + numcontext (future) - window_size = 2*self._model_feeder.numcontext+1 - features = np.lib.stride_tricks.as_strided( - features, - (num_strides, window_size, self._model_feeder.numcep), - (features.strides[0], features.strides[0], features.strides[1]), - writeable=False) - - # We add 1 to all elements of the transcript here to avoid any zero - # values since we use that as an end-of-sequence token for converting - # the batch into a SparseTensor. - try: - session.run(self._enqueue_op, feed_dict={ - self._model_feeder.ph_x: features, - self._model_feeder.ph_x_length: num_strides, - self._model_feeder.ph_y: transcript + 1, - self._model_feeder.ph_y_length: transcript_len - }) - except tf.errors.CancelledError: - return - - -class _TowerFeeder(object): - ''' - Internal class that represents a switchable input queue for one tower. - It creates, owns and combines three _DataSetLoader instances. - Keeps a ModelFeeder reference for accessing shared settings and placeholders. - ''' - def __init__(self, model_feeder, index, alphabet): - self._model_feeder = model_feeder - self.index = index - self._loaders = [_DataSetLoader(model_feeder, data_set, alphabet) for data_set in model_feeder.sets] - self._queues = [set_queue.queue for set_queue in self._loaders] - self._queue = tf.QueueBase.from_list(model_feeder.ph_queue_selector, self._queues) - self._close_op = self._queue.close(cancel_pending_enqueues=True) - - def next_batch(self): - ''' - Draw the next batch from from the combined switchable queue. - ''' - source, source_lengths, target, target_lengths = self._queue.dequeue_many(self._model_feeder.ph_batch_size) - # Back to sparse, then subtract one to get the real labels - sparse_labels = tf.contrib.layers.dense_to_sparse(target) - neg_ones = tf.SparseTensor(sparse_labels.indices, -1 * tf.ones_like(sparse_labels.values), sparse_labels.dense_shape) - return source, source_lengths, tf.sparse_add(sparse_labels, neg_ones) - - def start_queue_threads(self, session, coord): - ''' - Starts the queue threads of all owned _DataSetLoader instances. - ''' - queue_threads = [] - for set_queue in self._loaders: - queue_threads += set_queue.start_queue_threads(session, coord) - return queue_threads - - def close_queues(self, session): - ''' - Closes queues of all owned _DataSetLoader instances. - ''' - for set_queue in self._loaders: - set_queue.close_queue(session) +from functools import partial +from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio +from util.config import Config +from util.text import text_to_char_array + + +def read_csvs(csv_files): + source_data = None + for csv in csv_files: + file = pandas.read_csv(csv, encoding='utf-8', na_filter=False) + #FIXME: not cross-platform + csv_dir = os.path.dirname(os.path.abspath(csv)) + file['wav_filename'] = file['wav_filename'].str.replace(r'(^[^/])', lambda m: os.path.join(csv_dir, m.group(1))) + if source_data is None: + source_data = file + else: + source_data = source_data.append(file) + return source_data + + +def samples_to_mfccs(samples, sample_rate): + spectrogram = contrib_audio.audio_spectrogram(samples, window_size=512, stride=320, magnitude_squared=True) + mfccs = contrib_audio.mfcc(spectrogram, sample_rate, dct_coefficient_count=Config.n_input) + mfccs = tf.reshape(mfccs, [-1, Config.n_input]) + + return mfccs, tf.shape(mfccs)[0] + + +def audiofile_to_features(wav_filename): + samples = tf.read_file(wav_filename) + decoded = contrib_audio.decode_wav(samples, desired_channels=1) + features, features_len = samples_to_mfccs(decoded.audio, decoded.sample_rate) + + return features, features_len + + +def entry_to_features(wav_filename, transcript): + # https://bugs.python.org/issue32117 + features, features_len = audiofile_to_features(wav_filename) + return features, features_len, tf.SparseTensor(*transcript) + + +def to_sparse_tuple(sequence): + r"""Creates a sparse representention of ``sequence``. + Returns a tuple with (indices, values, shape) + """ + indices = np.asarray(list(zip([0]*len(sequence), range(len(sequence)))), dtype=np.int64) + shape = np.asarray([1, len(sequence)], dtype=np.int64) + return indices, sequence, shape + + +def create_dataset(csvs, batch_size, cache_path): + df = read_csvs(csvs) + df.sort_values(by='wav_filesize', inplace=True) + + num_batches = len(df) // batch_size + + # Convert to character index arrays + df['transcript'] = df['transcript'].apply(partial(text_to_char_array, alphabet=Config.alphabet)) + + def generate_values(): + for _, row in df.iterrows(): + yield row.wav_filename, to_sparse_tuple(row.transcript) + + # Batching a dataset of 2D SparseTensors creates 3D batches, which fail + # when passed to tf.nn.ctc_loss, so we reshape them to remove the extra + # dimension here. + def sparse_reshape(sparse): + shape = sparse.dense_shape + return tf.sparse.reshape(sparse, [shape[0], shape[2]]) + + def batch_fn(features, features_len, transcripts): + features = tf.data.Dataset.zip((features, features_len)) + features = features.padded_batch(batch_size, + padded_shapes=([None, Config.n_input], [])) + transcripts = transcripts.batch(batch_size).map(sparse_reshape) + return tf.data.Dataset.zip((features, transcripts)) + + num_gpus = len(Config.available_devices) + + dataset = (tf.data.Dataset.from_generator(generate_values, + output_types=(tf.string, (tf.int64, tf.int32, tf.int64))) + .map(entry_to_features, num_parallel_calls=tf.data.experimental.AUTOTUNE) + .cache(cache_path) + .window(batch_size, drop_remainder=True).flat_map(batch_fn) + .prefetch(num_gpus) + .repeat()) + return dataset, num_batches diff --git a/util/flags.py b/util/flags.py index f0bac5e4aa..1683e32bf4 100644 --- a/util/flags.py +++ b/util/flags.py @@ -10,9 +10,9 @@ def create_flags(): # Importer # ======== - tf.app.flags.DEFINE_string ('train_files', '', 'comma separated list of files specifying the dataset used for training. multiple files will get merged') - tf.app.flags.DEFINE_string ('dev_files', '', 'comma separated list of files specifying the dataset used for validation. multiple files will get merged') - tf.app.flags.DEFINE_string ('test_files', '', 'comma separated list of files specifying the dataset used for testing. multiple files will get merged') + tf.app.flags.DEFINE_string ('train_files', '', 'comma separated list of files specifying the dataset used for training. Multiple files will get merged. If empty, training will not be run.') + tf.app.flags.DEFINE_string ('dev_files', '', 'comma separated list of files specifying the dataset used for validation. Multiple files will get merged. If empty, validation will not be run.') + tf.app.flags.DEFINE_string ('test_files', '', 'comma separated list of files specifying the dataset used for testing. Multiple files will get merged. If empty, the model will not be tested.') tf.app.flags.DEFINE_boolean ('fulltrace', False, 'if full trace debug info should be generated during training') tf.app.flags.DEFINE_string ('train_cached_features_path', '', 'comma separated list of files specifying the dataset used for training. multiple files will get merged') @@ -22,8 +22,6 @@ def create_flags(): # Global Constants # ================ - tf.app.flags.DEFINE_boolean ('train', True, 'whether to train the network') - tf.app.flags.DEFINE_boolean ('test', True, 'whether to test the network') tf.app.flags.DEFINE_integer ('epoch', 75, 'target epoch to train - if negative, the absolute number of additional epochs will be trained') tf.app.flags.DEFINE_float ('dropout_rate', 0.05, 'dropout rate for feedforward layers') @@ -96,7 +94,7 @@ def create_flags(): # Early Stopping - tf.app.flags.DEFINE_boolean ('early_stop', True, 'enable early stopping mechanism over validation dataset') + tf.app.flags.DEFINE_boolean ('early_stop', True, 'enable early stopping mechanism over validation dataset. If validation is not being run, early stopping is disabled.') tf.app.flags.DEFINE_integer ('es_steps', 4, 'number of validations to consider for early stopping. Loss is not stored in the checkpoint so when checkpoint is revived it starts the loss calculation from start at that point') tf.app.flags.DEFINE_float ('es_mean_th', 0.5, 'mean threshold for loss to determine the condition if early stopping is required') tf.app.flags.DEFINE_float ('es_std_th', 0.5, 'standard deviation threshold for loss to determine the condition if early stopping is required') @@ -112,5 +110,5 @@ def create_flags(): # Inference mode - tf.app.flags.DEFINE_string ('one_shot_infer', '', 'one-shot inference mode: specify a wav file and the script will load the checkpoint and perform inference on it. Disables training, testing and exporting.') + tf.app.flags.DEFINE_string ('one_shot_infer', '', 'one-shot inference mode: specify a wav file and the script will load the checkpoint and perform inference on it.') diff --git a/util/preprocess.py b/util/preprocess.py deleted file mode 100644 index 1feb1ebbe8..0000000000 --- a/util/preprocess.py +++ /dev/null @@ -1,101 +0,0 @@ -import numpy as np -import os -import pandas -import tables - -from functools import partial -from multiprocessing.dummy import Pool -from util.audio import audiofile_to_input_vector -from util.text import text_to_char_array - -def pmap(fun, iterable): - pool = Pool() - results = pool.map(fun, iterable) - pool.close() - return results - - -def process_single_file(row, numcep, numcontext, alphabet): - # row = index, Series - _, file = row - features = audiofile_to_input_vector(file.wav_filename, numcep, numcontext) - features_len = len(features) - 2*numcontext - transcript = text_to_char_array(file.transcript, alphabet) - - if features_len < len(transcript): - raise ValueError('Error: Audio file {} is too short for transcription.'.format(file.wav_filename)) - - return features, features_len, transcript, len(transcript) - - -# load samples from CSV, compute features, optionally cache results on disk -def preprocess(csv_files, batch_size, numcep, numcontext, alphabet, hdf5_cache_path=None): - COLUMNS = ('features', 'features_len', 'transcript', 'transcript_len') - - print('Preprocessing', csv_files) - - if hdf5_cache_path and os.path.exists(hdf5_cache_path): - with tables.open_file(hdf5_cache_path, 'r') as file: - features = file.root.features[:] - features_len = file.root.features_len[:] - transcript = file.root.transcript[:] - transcript_len = file.root.transcript_len[:] - - # features are stored flattened, so reshape into [n_steps, numcep] - for i in range(len(features)): - features[i].shape = [features_len[i]+2*numcontext, numcep] - - in_data = list(zip(features, features_len, - transcript, transcript_len)) - print('Loaded from cache at', hdf5_cache_path) - return pandas.DataFrame(data=in_data, columns=COLUMNS) - - source_data = None - for csv in csv_files: - file = pandas.read_csv(csv, encoding='utf-8', na_filter=False) - #FIXME: not cross-platform - csv_dir = os.path.dirname(os.path.abspath(csv)) - file['wav_filename'] = file['wav_filename'].str.replace(r'(^[^/])', lambda m: os.path.join(csv_dir, m.group(1))) - if source_data is None: - source_data = file - else: - source_data = source_data.append(file) - - step_fn = partial(process_single_file, - numcep=numcep, - numcontext=numcontext, - alphabet=alphabet) - out_data = pmap(step_fn, source_data.iterrows()) - - if hdf5_cache_path: - print('Saving to', hdf5_cache_path) - - # list of tuples -> tuple of lists - features, features_len, transcript, transcript_len = zip(*out_data) - - with tables.open_file(hdf5_cache_path, 'w') as file: - features_dset = file.create_vlarray(file.root, - 'features', - tables.Float32Atom(), - filters=tables.Filters(complevel=1)) - # VLArray atoms need to be 1D, so flatten feature array - for f in features: - features_dset.append(np.reshape(f, -1)) - - features_len_dset = file.create_array(file.root, - 'features_len', - features_len) - - transcript_dset = file.create_vlarray(file.root, - 'transcript', - tables.Int32Atom(), - filters=tables.Filters(complevel=1)) - for t in transcript: - transcript_dset.append(t) - - transcript_len_dset = file.create_array(file.root, - 'transcript_len', - transcript_len) - - print('Preprocessing done') - return pandas.DataFrame(data=out_data, columns=COLUMNS)