/* * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ // // Implements helper functions and classes for intelligibility enhancement. // #include "webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h" #include #include #include #include using std::complex; using std::min; namespace webrtc { namespace intelligibility { float UpdateFactor(float target, float current, float limit) { float delta = fabsf(target - current); float sign = copysign(1.0f, target - current); return current + sign * fminf(delta, limit); } float AddDitherIfZero(float value) { return value == 0.f ? std::rand() * 0.01f / RAND_MAX : value; } complex zerofudge(complex c) { return complex(AddDitherIfZero(c.real()), AddDitherIfZero(c.imag())); } complex NewMean(complex mean, complex data, size_t count) { return mean + (data - mean) / static_cast(count); } void AddToMean(complex data, size_t count, complex* mean) { (*mean) = NewMean(*mean, data, count); } static const size_t kWindowBlockSize = 10; VarianceArray::VarianceArray(size_t num_freqs, StepType type, size_t window_size, float decay) : running_mean_(new complex[num_freqs]()), running_mean_sq_(new complex[num_freqs]()), sub_running_mean_(new complex[num_freqs]()), sub_running_mean_sq_(new complex[num_freqs]()), variance_(new float[num_freqs]()), conj_sum_(new float[num_freqs]()), num_freqs_(num_freqs), window_size_(window_size), decay_(decay), history_cursor_(0), count_(0), array_mean_(0.0f), buffer_full_(false) { history_.reset(new rtc::scoped_ptr[]>[num_freqs_]()); for (size_t i = 0; i < num_freqs_; ++i) { history_[i].reset(new complex[window_size_]()); } subhistory_.reset(new rtc::scoped_ptr[]>[num_freqs_]()); for (size_t i = 0; i < num_freqs_; ++i) { subhistory_[i].reset(new complex[window_size_]()); } subhistory_sq_.reset(new rtc::scoped_ptr[]>[num_freqs_]()); for (size_t i = 0; i < num_freqs_; ++i) { subhistory_sq_[i].reset(new complex[window_size_]()); } switch (type) { case kStepInfinite: step_func_ = &VarianceArray::InfiniteStep; break; case kStepDecaying: step_func_ = &VarianceArray::DecayStep; break; case kStepWindowed: step_func_ = &VarianceArray::WindowedStep; break; case kStepBlocked: step_func_ = &VarianceArray::BlockedStep; break; case kStepBlockBasedMovingAverage: step_func_ = &VarianceArray::BlockBasedMovingAverage; break; } } // Compute the variance with Welford's algorithm, adding some fudge to // the input in case of all-zeroes. void VarianceArray::InfiniteStep(const complex* data, bool skip_fudge) { array_mean_ = 0.0f; ++count_; for (size_t i = 0; i < num_freqs_; ++i) { complex sample = data[i]; if (!skip_fudge) { sample = zerofudge(sample); } if (count_ == 1) { running_mean_[i] = sample; variance_[i] = 0.0f; } else { float old_sum = conj_sum_[i]; complex old_mean = running_mean_[i]; running_mean_[i] = old_mean + (sample - old_mean) / static_cast(count_); conj_sum_[i] = (old_sum + std::conj(sample - old_mean) * (sample - running_mean_[i])) .real(); variance_[i] = conj_sum_[i] / (count_ - 1); } array_mean_ += (variance_[i] - array_mean_) / (i + 1); } } // Compute the variance from the beginning, with exponential decaying of the // series data. void VarianceArray::DecayStep(const complex* data, bool /*dummy*/) { array_mean_ = 0.0f; ++count_; for (size_t i = 0; i < num_freqs_; ++i) { complex sample = data[i]; sample = zerofudge(sample); if (count_ == 1) { running_mean_[i] = sample; running_mean_sq_[i] = sample * std::conj(sample); variance_[i] = 0.0f; } else { complex prev = running_mean_[i]; complex prev2 = running_mean_sq_[i]; running_mean_[i] = decay_ * prev + (1.0f - decay_) * sample; running_mean_sq_[i] = decay_ * prev2 + (1.0f - decay_) * sample * std::conj(sample); variance_[i] = (running_mean_sq_[i] - running_mean_[i] * std::conj(running_mean_[i])).real(); } array_mean_ += (variance_[i] - array_mean_) / (i + 1); } } // Windowed variance computation. On each step, the variances for the // window are recomputed from scratch, using Welford's algorithm. void VarianceArray::WindowedStep(const complex* data, bool /*dummy*/) { size_t num = min(count_ + 1, window_size_); array_mean_ = 0.0f; for (size_t i = 0; i < num_freqs_; ++i) { complex mean; float conj_sum = 0.0f; history_[i][history_cursor_] = data[i]; mean = history_[i][history_cursor_]; variance_[i] = 0.0f; for (size_t j = 1; j < num; ++j) { complex sample = zerofudge(history_[i][(history_cursor_ + j) % window_size_]); sample = history_[i][(history_cursor_ + j) % window_size_]; float old_sum = conj_sum; complex old_mean = mean; mean = old_mean + (sample - old_mean) / static_cast(j + 1); conj_sum = (old_sum + std::conj(sample - old_mean) * (sample - mean)).real(); variance_[i] = conj_sum / (j); } array_mean_ += (variance_[i] - array_mean_) / (i + 1); } history_cursor_ = (history_cursor_ + 1) % window_size_; ++count_; } // Variance with a window of blocks. Within each block, the variances are // recomputed from scratch at every stp, using |Var(X) = E(X^2) - E^2(X)|. // Once a block is filled with kWindowBlockSize samples, it is added to the // history window and a new block is started. The variances for the window // are recomputed from scratch at each of these transitions. void VarianceArray::BlockedStep(const complex* data, bool /*dummy*/) { size_t blocks = min(window_size_, history_cursor_ + 1); for (size_t i = 0; i < num_freqs_; ++i) { AddToMean(data[i], count_ + 1, &sub_running_mean_[i]); AddToMean(data[i] * std::conj(data[i]), count_ + 1, &sub_running_mean_sq_[i]); subhistory_[i][history_cursor_ % window_size_] = sub_running_mean_[i]; subhistory_sq_[i][history_cursor_ % window_size_] = sub_running_mean_sq_[i]; variance_[i] = (NewMean(running_mean_sq_[i], sub_running_mean_sq_[i], blocks) - NewMean(running_mean_[i], sub_running_mean_[i], blocks) * std::conj(NewMean(running_mean_[i], sub_running_mean_[i], blocks))) .real(); if (count_ == kWindowBlockSize - 1) { sub_running_mean_[i] = complex(0.0f, 0.0f); sub_running_mean_sq_[i] = complex(0.0f, 0.0f); running_mean_[i] = complex(0.0f, 0.0f); running_mean_sq_[i] = complex(0.0f, 0.0f); for (size_t j = 0; j < min(window_size_, history_cursor_); ++j) { AddToMean(subhistory_[i][j], j + 1, &running_mean_[i]); AddToMean(subhistory_sq_[i][j], j + 1, &running_mean_sq_[i]); } ++history_cursor_; } } ++count_; if (count_ == kWindowBlockSize) { count_ = 0; } } // Recomputes variances for each window from scratch based on previous window. void VarianceArray::BlockBasedMovingAverage(const std::complex* data, bool /*dummy*/) { // TODO(ekmeyerson) To mitigate potential divergence, add counter so that // after every so often sums are computed scratch by summing over all // elements instead of subtracting oldest and adding newest. for (size_t i = 0; i < num_freqs_; ++i) { sub_running_mean_[i] += data[i]; sub_running_mean_sq_[i] += data[i] * std::conj(data[i]); } ++count_; // TODO(ekmeyerson) Make kWindowBlockSize nonconstant to allow // experimentation with different block size,window size pairs. if (count_ >= kWindowBlockSize) { count_ = 0; for (size_t i = 0; i < num_freqs_; ++i) { running_mean_[i] -= subhistory_[i][history_cursor_]; running_mean_sq_[i] -= subhistory_sq_[i][history_cursor_]; float scale = 1.f / kWindowBlockSize; subhistory_[i][history_cursor_] = sub_running_mean_[i] * scale; subhistory_sq_[i][history_cursor_] = sub_running_mean_sq_[i] * scale; sub_running_mean_[i] = std::complex(0.0f, 0.0f); sub_running_mean_sq_[i] = std::complex(0.0f, 0.0f); running_mean_[i] += subhistory_[i][history_cursor_]; running_mean_sq_[i] += subhistory_sq_[i][history_cursor_]; scale = 1.f / (buffer_full_ ? window_size_ : history_cursor_ + 1); variance_[i] = std::real(running_mean_sq_[i] * scale - running_mean_[i] * scale * std::conj(running_mean_[i]) * scale); } ++history_cursor_; if (history_cursor_ >= window_size_) { buffer_full_ = true; history_cursor_ = 0; } } } void VarianceArray::Clear() { memset(running_mean_.get(), 0, sizeof(*running_mean_.get()) * num_freqs_); memset(running_mean_sq_.get(), 0, sizeof(*running_mean_sq_.get()) * num_freqs_); memset(variance_.get(), 0, sizeof(*variance_.get()) * num_freqs_); memset(conj_sum_.get(), 0, sizeof(*conj_sum_.get()) * num_freqs_); history_cursor_ = 0; count_ = 0; array_mean_ = 0.0f; } void VarianceArray::ApplyScale(float scale) { array_mean_ = 0.0f; for (size_t i = 0; i < num_freqs_; ++i) { variance_[i] *= scale * scale; array_mean_ += (variance_[i] - array_mean_) / (i + 1); } } GainApplier::GainApplier(size_t freqs, float change_limit) : num_freqs_(freqs), change_limit_(change_limit), target_(new float[freqs]()), current_(new float[freqs]()) { for (size_t i = 0; i < freqs; ++i) { target_[i] = 1.0f; current_[i] = 1.0f; } } void GainApplier::Apply(const complex* in_block, complex* out_block) { for (size_t i = 0; i < num_freqs_; ++i) { float factor = sqrtf(fabsf(current_[i])); if (!std::isnormal(factor)) { factor = 1.0f; } out_block[i] = factor * in_block[i]; current_[i] = UpdateFactor(target_[i], current_[i], change_limit_); } } } // namespace intelligibility } // namespace webrtc