test/AIAS
1
0
Fork 0
mirror of https://gitee.com/mymagicpower/AIAS.git synced 2024-11-29 18:58:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

no message

Browse Source
This commit is contained in:
Calvin 2021-09-21 14:04:12 +08:00
parent bc21789b34
commit a474e0ed7e
15 changed files with 1117 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
speech_sdks/fft_sdk/README.md
View File

@ -8,84 +8,34 @@
## 运行例子 - FFTExample
核心算法：
```text
/**
* Performs Fast Fourier Transformation in place.
*/
public void process(double[] signal) {
final int numPoints = signal.length;
// initialize real & imag array
real = signal;
imag = new double[numPoints];
...
String audioFilePath = "src/test/resources/audio.wav";
//-1 value implies the method to use default sample rate
int defaultSampleRate = -1;
//-1 value implies the method to process complete audio duration
int defaultAudioDuration = -1;
// perform FFT using the real & imag array
final double pi = Math.PI;
final int numStages = (int) (Math.log(numPoints) / Math.log(2));
final int halfNumPoints = numPoints >> 1;
int j = halfNumPoints;
// FFT time domain decomposition carried out by "bit reversal sorting"
// algorithm
int k;
for (int i = 1; i < numPoints - 2; i++) {
if (i < j) {
// swap
double tempReal = real[j];
double tempImag = imag[j];
real[j] = real[i];
imag[j] = imag[i];
real[i] = tempReal;
imag[i] = tempImag;
}
k = halfNumPoints;
while (k <= j) {
j -= k;
k >>= 1;
}
j += k;
}
JLibrosa jLibrosa = new JLibrosa();
// loop for each stage
for (int stage = 1; stage <= numStages; stage++) {
int LE = 1;
for (int i = 0; i < stage; i++) {
LE <<= 1;
}
final int LE2 = LE >> 1;
double UR = 1;
double UI = 0;
// calculate sine & cosine values
final double SR = Math.cos(pi / LE2);
final double SI = -Math.sin(pi / LE2);
// loop for each sub DFT
for (int subDFT = 1; subDFT <= LE2; subDFT++) {
// loop for each butterfly
for (int butterfly = subDFT - 1; butterfly <= numPoints - 1; butterfly += LE) {
int ip = butterfly + LE2;
// butterfly calculation
double tempReal = (double) (real[ip] * UR - imag[ip] * UI);
double tempImag = (double) (real[ip] * UI + imag[ip] * UR);
real[ip] = real[butterfly] - tempReal;
imag[ip] = imag[butterfly] - tempImag;
real[butterfly] += tempReal;
imag[butterfly] += tempImag;
}
// To read the magnitude values of audio files - equivalent to librosa.load('../audio.wav', sr=None)
float audioFeatureValues[] = jLibrosa.loadAndRead(audioFilePath, defaultSampleRate, defaultAudioDuration);
double tempUR = UR;
UR = tempUR * SR - UI * SI;
UI = tempUR * SI + UI * SR;
}
}
}
double[] arr = IntStream.range(0, audioFeatureValues.length).mapToDouble(i -> audioFeatureValues[i]).toArray();
double[] fft = FFT.fft(arr);
float[][] complex = FFT.rfft(fft);
System.out.println("Real parts: " + Arrays.toString(complex[0]));
System.out.println("Imaginary parts: " + Arrays.toString(complex[1]));
...
````
运行成功后，命令行应该看到下面的信息:
```text
# 复数的实部
Real parts: [0.002560000019002473, 8.446425149266073E-5, ..., 8.446425149266079E-5]
Real parts: [-1.8461201, -1.1128254, 0.58502156, 2.6774616, -1.7226994, ..., 0.15794027]
# 复数的虚部
Imaginary parts: [0.0, 8.056758864562464E-4, 4.0390382669844993E-4, ..., -8.056758864562461E-4]
Imaginary parts: [0.0, 1.2845019, 2.8104274, -1.3958083, 1.2868061, ..., -0.3447435, 0.0]
```

5
speech_sdks/fft_sdk/pom.xml
View File

@ -63,6 +63,11 @@
<artifactId>guava</artifactId>
<version>21.0</version>
</dependency>
<dependency>
<groupId>com.github.wendykierp</groupId>
<artifactId>JTransforms</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>jlibrosa</groupId>
<artifactId>jlibrosa</artifactId>

9
speech_sdks/fft_sdk/src/main/java/me/calvin/example/FFTExample.java
View File

@ -24,19 +24,20 @@ public class FFTExample {
// To read the magnitude values of audio files - equivalent to librosa.load('../audio.wav', sr=None)
float audioFeatureValues[] = jLibrosa.loadAndRead(audioFilePath, defaultSampleRate, defaultAudioDuration);
FFT fft = new FFT();
double[] arr = IntStream.range(0, audioFeatureValues.length).mapToDouble(i -> audioFeatureValues[i]).toArray();
// double[] arrTest = {0, 1, 0, 0};
//Arrays.copy for testing purpose
fft.process(Arrays.copyOf(arr, 32));
double[] fft = FFT.fft(arr);
float[][] complex = FFT.rfft(fft);
// int stride_size = 160;
// int window_size = 320;
// int truncate_size = (audioFeatureValues.length - window_size) % stride_size;
// int length = audioFeatureValues.length - truncate_size;
System.out.println("Real parts: " + Arrays.toString(fft.getReal()));
System.out.println("Imaginary parts: " + Arrays.toString(fft.getImag()));
System.out.println("Real parts: " + Arrays.toString(complex[0]));
System.out.println("Imaginary parts: " + Arrays.toString(complex[1]));
}

198
speech_sdks/fft_sdk/src/main/java/me/calvin/example/utils/FFT.java
View File

@ -1,135 +1,87 @@
/*
OC Volume - Java Speech Recognition Engine
Copyright (c) 2002-2004, OrangeCow organization
All rights reserved.
Redistribution and use in source and binary forms,
with or without modification, are permitted provided
that the following conditions are met:
* Redistributions of source code must retain the
above copyright notice, this list of conditions
and the following disclaimer.
* Redistributions in binary form must reproduce the
above copyright notice, this list of conditions
and the following disclaimer in the documentation
and/or other materials provided with the
distribution.
* Neither the name of the OrangeCow organization
nor the names of its contributors may be used to
endorse or promote products derived from this
software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS
AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
Contact information:
Please visit http://ocvolume.sourceforge.net.
*/
package me.calvin.example.utils;
import org.jtransforms.fft.DoubleFFT_1D;
/**
* Fast Fourier Transform.
*
* last updated on June 15, 2002<br>
* <b>description:</b> FFT class for real signals. Upon entry, N contains the
* numbers of points in the DFT, real[] and imaginary[] contain the real and
* imaginary parts of the input. Upon return, real[] and imaginary[] contain the
* DFT output. All signals run from 0 to N - 1<br>
* <b>input:</b> speech signal<br>
* <b>output:</b> real and imaginary part of DFT output
*
* @author Danny Su
* @author Hanns Holger Rutz
* A Fast Fourier Transform wrapper for jTransforms to provide similar functionality to numpy.fft
* functions used by the Blowhole Python implementation.
*/
public class FFT {
private double[] real;
private double[] imag;
public double[] getReal() {
return real;
}
/**
* Compute the fast fourier transform
*
* @param raw the raw signal
* @return the computed fast fourier transform
*/
public static double[] fft(double[] raw) {
double[] in = raw;
DoubleFFT_1D fft = new DoubleFFT_1D(in.length);
fft.realForward(in);
return in;
}
public double[] getImag() {
return imag;
}
/**
* Computes the physical layout of the fast fourier transform.
* See jTransform documentation for more information.
* http://incanter.org/docs/parallelcolt/api/edu/emory/mathcs/jtransforms/fft/DoubleFFT_1D.html#realForward(double[])
*
* @param fft the fast fourier transform
*/
public static float[][] rfft(double[] fft) {
float[][] result = null;
/**
* Performs Fast Fourier Transformation in place.
*/
public void process(double[] signal) {
final int numPoints = signal.length;
// initialize real & imag array
real = signal;
imag = new double[numPoints];
int n = fft.length;
if (n % 2 == 0) {
// n is even
result = new float[2][n / 2 + 1];
for (int i = 0; i < n / 2; i++) {
result[0][i] = (float) fft[2 * i]; //the real part fo the fast fourier transform
result[1][i] = (float) fft[2 * i + 1]; //the imaginary part of the fast fourier transform
}
result[1][0] = 0;
result[0][n / 2] = (float) fft[1];
} else {
// n is odd
result = new float[2][(n + 1) / 2];
for (int i = 0; i < n / 2; i++) {
result[0][i] = (float) fft[2 * i]; //the real part fo the fast fourier transform
result[1][i] = (float) fft[2 * i + 1]; //the imaginary part of the fast fourier transform
}
result[1][0] = 0;
result[1][(n - 1) / 2] = (float) fft[1];
// perform FFT using the real & imag array
final double pi = Math.PI;
final int numStages = (int) (Math.log(numPoints) / Math.log(2));
final int halfNumPoints = numPoints >> 1;
int j = halfNumPoints;
// FFT time domain decomposition carried out by "bit reversal sorting"
// algorithm
int k;
for (int i = 1; i < numPoints - 2; i++) {
if (i < j) {
// swap
double tempReal = real[j];
double tempImag = imag[j];
real[j] = real[i];
imag[j] = imag[i];
real[i] = tempReal;
imag[i] = tempImag;
}
k = halfNumPoints;
while (k <= j) {
j -= k;
k >>= 1;
}
j += k;
}
}
// loop for each stage
for (int stage = 1; stage <= numStages; stage++) {
int LE = 1;
for (int i = 0; i < stage; i++) {
LE <<= 1;
}
final int LE2 = LE >> 1;
double UR = 1;
double UI = 0;
// calculate sine & cosine values
final double SR = Math.cos(pi / LE2);
final double SI = -Math.sin(pi / LE2);
// loop for each sub DFT
for (int subDFT = 1; subDFT <= LE2; subDFT++) {
// loop for each butterfly
for (int butterfly = subDFT - 1; butterfly <= numPoints - 1; butterfly += LE) {
int ip = butterfly + LE2;
// butterfly calculation
double tempReal = (double) (real[ip] * UR - imag[ip] * UI);
double tempImag = (double) (real[ip] * UI + imag[ip] * UR);
real[ip] = real[butterfly] - tempReal;
imag[ip] = imag[butterfly] - tempImag;
real[butterfly] += tempReal;
imag[butterfly] += tempImag;
}
return result;
}
double tempUR = UR;
UR = tempUR * SR - UI * SI;
UI = tempUR * SI + UI * SR;
}
}
}
public static float[] abs(float[][] complex) {
float[] re = complex[0]; //the real part fo the fast fourier transform
float[] im = complex[1]; //the imaginary part of the fast fourier transform
float[] abs = new float[re.length];
for (int i = 0; i < re.length; i++) {
abs[i] = (float) Math.hypot(re[i], im[i]);
}
return abs;
}
/**
* Returns the Discrete Fourier Transform sample frequencies.
* See numpy.fft.rfftfreq for more information.
*
* @param n Window length
* @param d Sample spacing
* @return Array of length n + 1 containing the sample frequencies
*/
public static double[] rfftfreq(int n, double d) {
double val = 1.0 / (n * d);
int N = n / 2 + 1;
double[] results = new double[N];
for (int i = 0; i < N; i++) {
results[i] = i * val;
}
return results;
}
}

66
speech_sdks/ndarray_audio_sdk/README.md Normal file
View File

@ -0,0 +1,66 @@
## NDArray 高级音频工具箱
当前功能包括：
- 获取音频文件的float数组
- 保存音频文件
- 创建给定持续时间和采样率的静音音频段
- 在这个音频样本上加一段静音
- 将任意数量的语音片段连接在一起
- 生成以分贝为单位的音频均方根能量
- 将音频归一化，使其具有所需的有效值(以分贝为单位)
- 用快速傅里叶变换计算线性谱图
- Hanning窗
- 从wav提取mel频谱特征值
## 运行例子 - AudioExample
```text
...
NDManager manager = NDManager.newBaseManager(Device.cpu());
float[] floatArray = AudioArrayUtils.floatData("src/test/resources/test.wav");
//音频的float数组
logger.info("音频的float数组: {}", Arrays.toString(floatArray));
NDArray samples = manager.create(floatArray);
float rmsDb = AudioUtils.rmsDb(samples);
//返回以分贝为单位的音频均方根能量
logger.info("音频均方根能量: {}", rmsDb);
//提取特征前将音频归一化至-20 dB(以分贝为单位)
float target_dB = -20f;
samples = AudioUtils.normalize(samples, target_dB);
System.out.println("音频归一化: " + samples.toDebugString(1000000000, 1000, 1000, 1000));
//生成帧的跨步大小(以毫秒为单位)
float stride_ms = 10f;
//用于生成帧的窗口大小(毫秒)
float window_ms = 20f;
samples = AudioUtils.linearSpecgram(manager, samples, stride_ms, window_ms);
logger.info("快速傅里叶变换计算线性谱图: {}", samples.toDebugString(1000000000, 1000, 10, 1000));...
```
命令行查看输出：
```text
...
[INFO ] - 音频均方根能量: -28.989937
[INFO ] - 音频归一化: ND: (134240) cpu() float32
[ 3.09278257e-03, 3.26460390e-03, 1.71821259e-04, ... 133240 more]
[INFO ] - 快速傅里叶变换计算线性谱图: ND: (161, 838) cpu() float32
[[-15.4571, -16.4412, -16.7098, -20.372 , -23.9935, -15.8598, -17.1589, -15.5935, ..., -14.3427],
[-15.948 , -16.8391, -16.8302, -17.8034, -19.115 , -15.8378, -19.4812, -15.7247, ..., -14.7543],
[-20.8405, -18.5733, -19.4289, -19.1861, -19.4255, -18.1996, -18.0149, -18.977 , ..., -17.5405],
[-19.1938, -21.0139, -21.07 , -20.2931, -20.23 , -22.3037, -20.1103, -21.3521, ..., -20.2267],
[-19.7823, -21.2425, -21.5705, -19.9856, -21.6053, -20.9323, -22.4014, -21.5406, ..., -20.1177],
[-20.0329, -23.9688, -20.718 , -20.9419, -23.5446, -21.1718, -22.1597, -20.9377, ..., -21.3833],
[-19.3693, -21.0484, -21.1794, -20.765 , -20.6318, -20.5121, -21.7306, -20.6366, ..., -21.5107],
[-18.6552, -20.0077, -20.6954, -20.5476, -19.7953, -21.1081, -22.0988, -20.7157, ..., -20.6352],
[-18.9167, -19.3219, -20.1954, -24.7476, -20.7662, -20.3794, -24.4699, -22.1381, ..., -22.1803],
[-19.3055, -19.4264, -20.4486, -22.8299, -21.0847, -23.5101, -20.4897, -19.7943, ..., -22.8922],
... 151 more]
...
```

BIN
speech_sdks/ndarray_audio_sdk/lib/jlibrosa-1.1.8-SNAPSHOT.jar Normal file
View File

Binary file not shown.

222
speech_sdks/ndarray_audio_sdk/ndarray_audio_sdk.iml Normal file
View File

@ -0,0 +1,222 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="CheckStyle-IDEA-Module">
<option name="configuration">
<map />
</option>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" type="java-test-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="aias-jieba-lib-0.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-slf4j-impl:2.12.1" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.12.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.apache.logging.log4j:log4j-core:2.12.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.12.0" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:21.0" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacv-platform:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacv:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:videoinput:0.200-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas-platform:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp-platform:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:android-arm:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:android-arm64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:android-x86:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:android-x86_64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:ios-arm64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:ios-x86_64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:linux-armhf:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:linux-arm64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:linux-ppc64le:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:linux-x86:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:linux-x86_64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:macosx-x86_64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:windows-x86:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:javacpp:windows-x86_64:1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:android-arm:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:android-arm64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:android-x86:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:android-x86_64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:ios-arm64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:ios-x86_64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:linux-x86:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:linux-x86_64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:linux-armhf:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:linux-arm64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:linux-ppc64le:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:macosx-x86_64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:windows-x86:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:openblas:windows-x86_64:0.3.10-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv-platform:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:android-arm:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:android-arm64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:android-x86:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:android-x86_64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:ios-arm64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:ios-x86_64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:linux-x86:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:linux-x86_64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:linux-armhf:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:linux-arm64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:linux-ppc64le:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:macosx-x86_64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:windows-x86:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:opencv:windows-x86_64:4.4.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg-platform:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:android-arm:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:android-arm64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:android-x86:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:android-x86_64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:linux-x86:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:linux-x86_64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:linux-armhf:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:linux-arm64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:linux-ppc64le:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:macosx-x86_64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:windows-x86:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:ffmpeg:windows-x86_64:4.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture-platform:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:linux-x86:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:linux-x86_64:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:linux-armhf:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:linux-arm64:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:windows-x86:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flycapture:windows-x86_64:2.13.3.31-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394-platform:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:linux-x86:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:linux-x86_64:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:linux-armhf:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:linux-arm64:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:linux-ppc64le:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:macosx-x86_64:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:windows-x86:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libdc1394:windows-x86_64:2.2.6-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect-platform:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:linux-x86:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:linux-x86_64:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:linux-armhf:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:linux-arm64:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:linux-ppc64le:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:macosx-x86_64:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:windows-x86:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect:windows-x86_64:0.5.7-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2-platform:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2:linux-x86:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2:linux-x86_64:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2:macosx-x86_64:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:libfreenect2:windows-x86_64:0.2.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense-platform:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:linux-x86:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:linux-x86_64:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:macosx-x86_64:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:windows-x86:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense:windows-x86_64:1.12.4-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2-platform:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:linux-x86:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:linux-x86_64:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:macosx-x86_64:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:windows-x86:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:librealsense2:windows-x86_64:2.29.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:videoinput-platform:0.200-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:videoinput:windows-x86:0.200-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:videoinput:windows-x86_64:0.200-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus-platform:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:android-arm:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:android-arm64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:android-x86:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:android-x86_64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:linux-x86:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:linux-x86_64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:linux-armhf:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:linux-arm64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:linux-ppc64le:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:macosx-x86_64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:windows-x86:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:artoolkitplus:windows-x86_64:2.3.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark-platform:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:android-arm:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:android-arm64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:android-x86:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:android-x86_64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:linux-x86:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:linux-x86_64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:linux-armhf:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:linux-arm64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:linux-ppc64le:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:macosx-x86_64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:windows-x86:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:flandmark:windows-x86_64:1.07-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica-platform:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:android-arm:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:android-arm64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:android-x86:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:android-x86_64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:linux-x86:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:linux-x86_64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:linux-armhf:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:linux-arm64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:linux-ppc64le:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:macosx-x86_64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:windows-x86:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:leptonica:windows-x86_64:1.80.0-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract-platform:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:android-arm:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:android-arm64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:android-x86:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:android-x86_64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:linux-x86:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:linux-x86_64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:linux-armhf:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:linux-arm64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:linux-ppc64le:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:macosx-x86_64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:windows-x86:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: org.bytedeco:tesseract:windows-x86_64:4.1.1-1.5.4" level="project" />
<orderEntry type="library" name="Maven: ai.djl:api:0.12.0" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.7" level="project" />
<orderEntry type="library" name="Maven: net.java.dev.jna:jna:5.8.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.20" level="project" />
<orderEntry type="library" name="Maven: ai.djl:basicdataset:0.12.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-csv:1.8" level="project" />
<orderEntry type="library" name="Maven: ai.djl:model-zoo:0.12.0" level="project" />
<orderEntry type="library" name="Maven: ai.djl.mxnet:mxnet-model-zoo:0.12.0" level="project" />
<orderEntry type="library" name="Maven: ai.djl.mxnet:mxnet-engine:0.12.0" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: ai.djl.mxnet:mxnet-native-auto:1.8.0" level="project" />
<orderEntry type="library" name="Maven: com.github.wendykierp:JTransforms:3.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-math3:3.5" level="project" />
<orderEntry type="library" name="Maven: pl.edu.icm:JLargeArrays:1.5" level="project" />
<orderEntry type="module-library">
<library name="Maven: jlibrosa:jlibrosa:1.1.8-SNAPSHOT">
<CLASSES>
<root url="jar://$MODULE_DIR$/lib/jlibrosa-1.1.8-SNAPSHOT.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />
</library>
</orderEntry>
</component>
</module>

119
speech_sdks/ndarray_audio_sdk/pom.xml Normal file
View File

@ -0,0 +1,119 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>calvin</groupId>
<artifactId>librosa-sdk</artifactId>
<version>0.1</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<djl.version>0.12.0</djl.version>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
<version>3.8.1</version>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.12.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>21.0</version>
</dependency>
<dependency>
<groupId>org.bytedeco</groupId>
<artifactId>javacv-platform</artifactId>
<version>1.5.4</version>
</dependency>
<!-- 服务器端推理引擎 -->
<dependency>
<groupId>ai.djl</groupId>
<artifactId>api</artifactId>
<version>${djl.version}</version>
</dependency>
<dependency>
<groupId>ai.djl</groupId>
<artifactId>basicdataset</artifactId>
<version>${djl.version}</version>
</dependency>
<dependency>
<groupId>ai.djl</groupId>
<artifactId>model-zoo</artifactId>
<version>${djl.version}</version>
</dependency>
<!-- MXNet -->
<dependency>
<groupId>ai.djl.mxnet</groupId>
<artifactId>mxnet-model-zoo</artifactId>
<version>${djl.version}</version>
</dependency>
<dependency>
<groupId>ai.djl.mxnet</groupId>
<artifactId>mxnet-engine</artifactId>
<version>${djl.version}</version>
</dependency>
<dependency>
<groupId>ai.djl.mxnet</groupId>
<artifactId>mxnet-native-auto</artifactId>
<version>1.8.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.github.wendykierp</groupId>
<artifactId>JTransforms</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>jlibrosa</groupId>
<artifactId>jlibrosa</artifactId>
<version>1.1.8-SNAPSHOT</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/jlibrosa-1.1.8-SNAPSHOT.jar</systemPath>
</dependency>
</dependencies>
</project>

40
speech_sdks/ndarray_audio_sdk/src/main/java/me/calvin/aias/AudioExample.java Normal file
View File

@ -0,0 +1,40 @@
package me.calvin.aias;
import ai.djl.Device;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDManager;
import me.calvin.aias.util.AudioArrayUtils;
import me.calvin.aias.util.AudioUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
public class AudioExample {
private static final Logger logger = LoggerFactory.getLogger(AudioExample.class);
// 测试
public static void main(String[] args) throws Exception {
NDManager manager = NDManager.newBaseManager(Device.cpu());
float[] floatArray = AudioArrayUtils.floatData("src/test/resources/test.wav");
//音频的float数组
logger.info("音频的float数组: {}", Arrays.toString(floatArray));
NDArray samples = manager.create(floatArray);
float rmsDb = AudioUtils.rmsDb(samples);
//返回以分贝为单位的音频均方根能量
logger.info("音频均方根能量: {}", rmsDb);
//提取特征前将音频归一化至-20 dB(以分贝为单位)
float target_dB = -20f;
samples = AudioUtils.normalize(samples, target_dB);
System.out.println("音频归一化: " + samples.toDebugString(1000000000, 1000, 1000, 1000));
//生成帧的跨步大小(以毫秒为单位)
float stride_ms = 10f;
//用于生成帧的窗口大小(毫秒)
float window_ms = 20f;
samples = AudioUtils.linearSpecgram(manager, samples, stride_ms, window_ms);
logger.info("快速傅里叶变换计算线性谱图: {}", samples.toDebugString(1000000000, 1000, 10, 1000));
}
}

218
speech_sdks/ndarray_audio_sdk/src/main/java/me/calvin/aias/util/AudioArrayUtils.java Normal file
View File

@ -0,0 +1,218 @@
package me.calvin.aias.util;
import ai.djl.ndarray.NDList;
import org.bytedeco.javacv.*;
import javax.sound.sampled.AudioFileFormat;
import javax.sound.sampled.AudioFormat;
import javax.sound.sampled.AudioInputStream;
import javax.sound.sampled.AudioSystem;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.nio.Buffer;
import java.nio.ByteBuffer;
import java.nio.ShortBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* 获取音频数组
*
* @author Calvin
*/
public class AudioArrayUtils {
public static void main(String[] args) throws FrameGrabber.Exception {
System.out.println(Arrays.toString(AudioArrayUtils.floatData("src/test/resources/test.wav")));
}
private static final class FrameData {
public final Buffer[] samples;
public final Integer sampleRate;
public final Integer audioChannels;
public FrameData(Buffer[] samples, Integer sampleRate, Integer audioChannels) {
this.samples = samples;
this.sampleRate = sampleRate;
this.audioChannels = audioChannels;
}
}
/**
* 获取音频文件的float数组
* Read and convert audio file to float32.
*
* @param path
* @return
* @throws FrameGrabber.Exception
*/
public static float[] floatData(String path) throws FrameGrabber.Exception {
// Audio sample type is usually integer or float-point.
// Integers will be scaled to [-1, 1] in float32.
float scale = (float) 1.0 / Float.valueOf(1 << ((8 * 2) - 1));
List<Float> floatList = new ArrayList<>();
try (FFmpegFrameGrabber audioGrabber = new FFmpegFrameGrabber(path)) {
try {
audioGrabber.start();
Frame frame;
while ((frame = audioGrabber.grabFrame()) != null) {
Buffer[] buffers = frame.samples;
Buffer[] copiedBuffers = new Buffer[buffers.length];
for (int i = 0; i < buffers.length; i++) {
deepCopy((ShortBuffer) buffers[i], (ShortBuffer) copiedBuffers[i]);
}
ShortBuffer sb = (ShortBuffer) buffers[0];
for (int i = 0; i < sb.limit(); i++) {
floatList.add(new Float(sb.get() * scale));
}
}
} catch (FrameGrabber.Exception e) {
e.printStackTrace();
}
float[] floatArray = new float[floatList.size()];
int i = 0;
for (Float f : floatList) {
floatArray[i++] = (f != null ? f : Float.NaN); // Or whatever default you want.
}
return floatArray;
}
}
/**
* Deep copy - shortBuffer
*
* @param source
* @param target
* @return
*/
private static ShortBuffer deepCopy(ShortBuffer source, ShortBuffer target) {
int sourceP = source.position();
int sourceL = source.limit();
if (null == target) {
target = ShortBuffer.allocate(source.remaining());
}
target.put(source);
target.flip();
source.position(sourceP);
source.limit(sourceL);
return target;
}
/**
* Deep copy - byteBuffer
*
* @param source
* @param target
* @return
*/
private static ByteBuffer deepCopy(ByteBuffer source, ByteBuffer target) {
int sourceP = source.position();
int sourceL = source.limit();
if (null == target) {
target = ByteBuffer.allocate(source.remaining());
}
target.put(source);
target.flip();
source.position(sourceP);
source.limit(sourceL);
return target;
}
/**
* 获取音频文件的FrameData列表
*
* @param path
* @return
* @throws FrameGrabber.Exception
*/
public static List<FrameData> frameData(String path) throws FrameGrabber.Exception {
//frameRecorder setup during initialization
List<FrameData> audioData = new ArrayList<>();
try (FFmpegFrameGrabber audioGrabber = new FFmpegFrameGrabber(path)) {
try {
audioGrabber.start();
Frame frame;
while ((frame = audioGrabber.grabFrame()) != null) {
Buffer[] buffers = frame.samples;
Buffer[] copiedBuffers = new Buffer[buffers.length];
for (int i = 0; i < buffers.length; i++) {
deepCopy((ShortBuffer) buffers[i], (ShortBuffer) copiedBuffers[i]);
}
FrameData frameData = new FrameData(copiedBuffers, frame.sampleRate, frame.audioChannels);
audioData.add(frameData);
}
} catch (FrameGrabber.Exception e) {
e.printStackTrace();
}
return audioData;
}
}
/**
* 保存音频文件
*
* @param buffer
* @param sampleRate
* @param audioChannels
* @param outs
* @throws Exception
*/
public static void toWavFile(float[] buffer, float sampleRate, int audioChannels, File outs) throws Exception {
if (sampleRate == 0.0) {
sampleRate = 22050;
}
if (audioChannels == 0) {
audioChannels = 1;
}
final byte[] byteBuffer = new byte[buffer.length * 2];
int bufferIndex = 0;
for (int i = 0; i < byteBuffer.length; i++) {
final int x = (int) (buffer[bufferIndex++]); //* 32767.0
byteBuffer[i++] = (byte) x;
byteBuffer[i] = (byte) (x >>> 8);
}
AudioFormat format = new AudioFormat(sampleRate, 16, audioChannels, true, false);
try (ByteArrayInputStream bais = new ByteArrayInputStream(byteBuffer);
AudioInputStream audioInputStream = new AudioInputStream(bais, format, buffer.length)) {
AudioSystem.write(audioInputStream, AudioFileFormat.Type.WAVE, outs);
}
}
/**
* 保存音频文件
*
* @param audioData
* @param path
* @param audioChannels
*/
public void toWavFile(List<FrameData> audioData, String path, int audioChannels) {
try (FFmpegFrameRecorder audioGrabber = new FFmpegFrameRecorder(path, audioChannels)) {
for (FrameData frameData : audioData) {
Frame frame = new Frame();
frame.sampleRate = frameData.sampleRate;
frame.audioChannels = frameData.audioChannels;
frame.samples = frameData.samples;
audioGrabber.record(frame);
}
} catch (FrameRecorder.Exception e) {
}
}
}

241
speech_sdks/ndarray_audio_sdk/src/main/java/me/calvin/aias/util/AudioUtils.java Normal file
View File

@ -0,0 +1,241 @@
package me.calvin.aias.util;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDArrays;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.ndarray.types.Shape;
import com.jlibrosa.audio.JLibrosa;
public class AudioUtils {
static int mel_window_step = 10;
static float max_gain_db = 300.0f;
static int sample_rate = 16000;
static float eps = 1e-14f;
/**
* 创建给定持续时间和采样率的静音音频段
*
* @param manager
* @param duration : 静音音频段长度，单位 second
* @param sampleRate : 采样率
* @return
*/
public static NDArray makeSilence(NDManager manager, long duration, int sampleRate) {
NDArray samples = manager.zeros(new Shape(duration * sampleRate));
return samples;
}
/**
* 在这个音频样本上加一段静音
*
* @param manager
* @param wav
* @param padLength
* @param sides : padding 位置:
* 'beginning' - 增加静音片段到开头
* 'end' - 增加静音片段到末尾
* 'both' - 两边都增加静音片段
* @return
* @throws Exception
*/
public static NDArray padSilence(NDManager manager, NDArray wav, long padLength, String sides) throws Exception {
NDArray pad = manager.zeros(new Shape(padLength));
if (sides.equals("beginning")) {
wav = pad.concat(wav);
} else if (sides.equals("end")) {
wav = wav.concat(pad);
} else if (sides.equals("both")) {
wav = pad.concat(wav);
wav = wav.concat(pad);
} else {
throw new Exception("Unknown value for the sides " + sides);
}
return wav;
}
/**
* 将任意数量的语音片段连接在一起
*
* @param segments : 要连接的输入语音片段
* @return
*/
public static NDArray concatenate(NDList segments) {
NDArray array = segments.get(0);
for (int i = 1; i < segments.size(); i++) {
array = array.concat(segments.get(i));
}
return array;
}
/**
* 生成以分贝为单位的音频均方根能量
* Root mean square energy in decibels.
*
* @param samples
* @return
*/
public static float rmsDb(NDArray samples) {
samples = samples.pow(2);
samples = samples.mean();
samples = samples.log10().mul(10);
return samples.toFloatArray()[0];
}
/**
* 将音频归一化，使其具有所需的有效值(以分贝为单位)
* Target RMS value in decibels. This value should be
* less than 0.0 as 0.0 is full-scale audio.
*
* @param samples
* @param target_db
* @return
* @throws Exception
*/
public static NDArray normalize(NDArray samples, float target_db) {
float gain = target_db - rmsDb(samples);
gain = Math.min(gain, max_gain_db);
//对音频施加分贝增益
//Gain in decibels to apply to samples
float factor = (float) Math.pow(10f, gain / 20f);
samples = samples.mul(factor);
return samples;
}
/**
* 用快速傅里叶变换计算线性谱图
*
* @param manager
* @param samples
* @param stride_ms
* @param window_ms
* @return
*/
public static NDArray linearSpecgram(NDManager manager, NDArray samples, float stride_ms, float window_ms) {
int strideSize = (int) (0.001 * sample_rate * stride_ms);
int windowSize = (int) (0.001 * sample_rate * window_ms);
long truncateSize = (samples.size() - windowSize) % strideSize;
long len = samples.size() - truncateSize;
samples = samples.get(":" + len);
//Shape nshape = new Shape(windowSize, (samples.size() - windowSize) / strideSize + 1); // 320 ,838
//nstrides = (samples.strides[0], samples.strides[0] * stride_size)
//strides[0] = 4 个字节, 由于已经转为float类型，所以对应当前samples中一个元素
//np.lib.stride_tricks.as_strided(samples, shape=nshape, strides=nstrides)
int rows = windowSize; //320
int cols = ((int) samples.size() - windowSize) / strideSize + 1; //838
float[] floatArray = samples.toFloatArray();
float[][] windows = new float[rows][cols];
for (int row = 0; row < rows; row++) {
for (int col = 0; col < cols; col++) {
windows[row][col] = floatArray[row + col * strideSize];
}
}
// 快速傅里叶变换
float[] weighting = hanningWindow(windowSize);
for (int row = 0; row < rows; row++) {
for (int col = 0; col < cols; col++) {
windows[row][col] = windows[row][col] * weighting[row];
}
}
double[] arr = null;
NDList fftList = new NDList();
for (int col = 0; col < cols; col++) {
arr = new double[rows];
for (int row = 0; row < rows; row++) {
arr[row] = windows[row][col];
}
double[] fft = FFT.fft(arr);
float[][] complex = FFT.rfft(fft);
NDArray array = manager.create(FFT.abs(complex));
fftList.add(array);
}
NDArray fft = NDArrays.stack(fftList).transpose();
fft = fft.pow(2);
NDArray weightingArray = manager.create(weighting);
weightingArray = weightingArray.pow(2);
NDArray scale = weightingArray.sum().mul(sample_rate);
NDArray middle = fft.get("1:-1,:");
middle = middle.mul(2).div(scale);
NDArray head = fft.get("0,:").div(scale).reshape(1, fft.getShape().get(1));
NDArray tail = fft.get("-1,:").div(scale).reshape(1, fft.getShape().get(1));
NDList list = new NDList(head, middle, tail);
fft = NDArrays.concat(list, 0);
NDArray freqsArray = manager.arange(fft.getShape().get(0));
freqsArray = freqsArray.mul(sample_rate / windowSize);
float[] freqs = freqsArray.toFloatArray();
int ind = 0;
for (int i = 0; i < freqs.length; i++) {
if (freqs[i] <= (sample_rate / 2)) {
ind = i;
} else {
break;
}
}
ind = ind + 1;
fft = fft.get(":" + ind + ",:").add(eps);
fft = fft.log();
// System.out.println(fft.toDebugString(1000000000, 1000, 10, 1000));
return fft;
}
/**
* Hanning窗
* The Hanning window is a taper formed by using a weighted cosine.
*
* @param size
* @return
*/
public static float[] hanningWindow(int size) {
float[] data = new float[size];
for (int n = 1; n < size; n++) {
data[n] = (float) (0.5 * (1 - Math.cos((2 * Math.PI * n)
/ (size - 1))));
}
return data;
}
/**
* Hanning窗
* The Hanning window is a taper formed by using a weighted cosine.
*
* @param recordedData
* @return
*/
public static float[] hanningWindow(float[] recordedData) {
for (int n = 1; n < recordedData.length; n++) {
recordedData[n] *= 0.5 * (1 - Math.cos((2 * Math.PI * n)
/ (recordedData.length - 1)));
}
return recordedData;
}
/**
* 从wav提取mel频谱特征值
*
* @param samples
* @param n_fft 1024
* @param n_mels 40
* @return
*/
public static float[][] melSpecgram(NDArray samples, int n_fft, int n_mels) {
JLibrosa librosa = new JLibrosa();
float[][] melSpectrogram =
librosa.generateMelSpectroGram(samples.toFloatArray(), sample_rate, n_fft, n_mels, (sample_rate * mel_window_step / 1000));
return melSpectrogram;
}
}

91
speech_sdks/ndarray_audio_sdk/src/main/java/me/calvin/aias/util/FFT.java Normal file
View File

@ -0,0 +1,91 @@
package me.calvin.aias.util;
import org.jtransforms.fft.DoubleFFT_1D;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* A Fast Fourier Transform wrapper for jTransforms to provide similar functionality to numpy.fft
* functions used by the Blowhole Python implementation.
*/
public class FFT {
/**
* Compute the fast fourier transform
*
* @param raw the raw signal
* @return the computed fast fourier transform
*/
public static double[] fft(double[] raw) {
double[] in = raw;
DoubleFFT_1D fft = new DoubleFFT_1D(in.length);
fft.realForward(in);
return in;
}
/**
* Computes the physical layout of the fast fourier transform.
* See jTransform documentation for more information.
* http://incanter.org/docs/parallelcolt/api/edu/emory/mathcs/jtransforms/fft/DoubleFFT_1D.html#realForward(double[])
*
* @param fft the fast fourier transform
*/
public static float[][] rfft(double[] fft) {
float[][] result = null;
int n = fft.length;
if (n % 2 == 0) {
// n is even
result = new float[2][n / 2 + 1];
for (int i = 0; i < n / 2; i++) {
result[0][i] = (float) fft[2 * i]; //the real part fo the fast fourier transform
result[1][i] = (float) fft[2 * i + 1]; //the imaginary part of the fast fourier transform
}
result[1][0] = 0;
result[0][n / 2] = (float) fft[1];
} else {
// n is odd
result = new float[2][(n + 1) / 2];
for (int i = 0; i < n / 2; i++) {
result[0][i] = (float) fft[2 * i]; //the real part fo the fast fourier transform
result[1][i] = (float) fft[2 * i + 1]; //the imaginary part of the fast fourier transform
}
result[1][0] = 0;
result[1][(n - 1) / 2] = (float) fft[1];
}
return result;
}
public static float[] abs(float[][] complex) {
float[] re = complex[0]; //the real part fo the fast fourier transform
float[] im = complex[1]; //the imaginary part of the fast fourier transform
float[] abs = new float[re.length];
for (int i = 0; i < re.length; i++) {
abs[i] = (float) Math.hypot(re[i], im[i]);
}
return abs;
}
/**
* Returns the Discrete Fourier Transform sample frequencies.
* See numpy.fft.rfftfreq for more information.
*
* @param n Window length
* @param d Sample spacing
* @return Array of length n + 1 containing the sample frequencies
*/
public static double[] rfftfreq(int n, double d) {
double val = 1.0 / (n * d);
int N = n / 2 + 1;
double[] results = new double[N];
for (int i = 0; i < N; i++) {
results[i] = i * val;
}
return results;
}
}

17
speech_sdks/ndarray_audio_sdk/src/main/resources/log4j2.xml Normal file
View File

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Appenders>
<Console name="console" target="SYSTEM_OUT">
<PatternLayout
pattern="[%-5level] - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="info" additivity="false">
<AppenderRef ref="console"/>
</Root>
<Logger name="me.calvin" level="${sys:me.calvin.logging.level:-info}" additivity="false">
<AppenderRef ref="console"/>
</Logger>
</Loggers>
</Configuration>

BIN
speech_sdks/ndarray_audio_sdk/src/test/resources/audio.wav Normal file
View File

Binary file not shown.

BIN
speech_sdks/ndarray_audio_sdk/src/test/resources/test.wav Normal file
View File

Binary file not shown.