ppgia/DIA.X540.cloud-ai.p3-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adds STT functionality with audio upload

Browse Source 
Adds speech-to-text (STT) functionality by allowing users to upload audio clips, start transcription jobs, and download transcriptions.

Introduces new API endpoints for STT upload, start, and download.

Also, converts AudioClip to WAV byte array.
This commit is contained in:
Jonas Luz Jr.
2025-11-26 19:31:12 -03:00
parent ea4535ebb6
commit e0daa00205
6 changed files with 447 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assets/_Client/Scripts/Core/AppManager.cs
+109 -35
View File
@@ -7,6 +7,10 @@ namespace PPGIA.X540.Project3
{
public class AppManager : MonoBehaviour
{
// Singleton instance
public static AppManager Instance { get; private set; }
#region -- Fields & Properties ----------------------------------------
[Header("References")]
[SerializeField]
private UIController _uiController;
@@ -14,59 +18,129 @@ namespace PPGIA.X540.Project3
[SerializeField]
private ApiClientManager _apiManager;
[SerializeField]
private AudioCapture _audioCapture;
private AudioClip _recordedClip;
#endregion ------------------------------------------------------------
#region -- MonoBehaviour Methods --------------------------------------
private void Awake()
{
if (_uiController == null)
_uiController = GetComponent<UIController>();
if (_apiManager == null)
_apiManager = GetComponent<ApiClientManager>();
}
// Singleton pattern implementation
if (Instance == null)
{
Instance = this;
DontDestroyOnLoad(gameObject);
}
else
{
Destroy(gameObject);
return;
}
if (_uiController == null)
{
Debug.LogError("UIController reference is missing in AppManager.");
}
if (_apiManager == null)
{
Debug.LogError("ApiClientManager reference is missing in AppManager.");
}
if (_audioCapture == null)
{
Debug.LogError("AudioCapture reference is missing in AppManager.");
}
_uiController.OnTalkButtonClicked += HandleTalkButtonClicked;
_audioCapture.OnRecordingSaved += HandleClipSaved;
void Start()
{
_apiManager.CloseSession(
() => _uiController.SessionActive = _apiManager.IsSessionActive);
}
private void OnEnable()
{
_uiController.OnSessionButtonClicked += HandleSessionButtonClicked;
_uiController.OnSendChatButtonClicked += HandleSendChatButtonClicked;
_apiManager.InitiateSession(() =>
{
Debug.Log("API session initiated successfully.");
});
}
private void OnDisable()
void OnDisable()
{
_uiController.OnSessionButtonClicked -= HandleSessionButtonClicked;
_uiController.OnSendChatButtonClicked -= HandleSendChatButtonClicked;
if (_apiManager != null && _apiManager.IsSessionActive)
{
_apiManager.CloseSession(() =>
{
Debug.Log("API session closed successfully.");
});
}
}
private void HandleSessionButtonClicked()
private void OnDestroy()
{
_uiController.OnTalkButtonClicked -= HandleTalkButtonClicked;
_audioCapture.OnRecordingSaved -= HandleClipSaved;
}
#endregion ------------------------------------------------------------
private void HandleClipSaved(string filePath)
{
Debug.Log($"Audio clip saved at: {filePath}");
_apiManager.UploadAudioClip(
filePath,
(s3Key) =>
{
Debug.Log($"Clip uploaded to: {s3Key}");
_apiManager.StartTranscript(
s3Key,
(jobName) =>
{
Debug.Log($"Transcription job started: {jobName}");
_apiManager.DownloadTranscription(jobName, (transcript) =>
{
Debug.Log($"Transcription completed: {transcript}");
_uiController.AppendChatOutput($"\nUser: {transcript}\n");
_apiManager.SendChatMessage(transcript,
(response) =>
{
_uiController.AppendChatOutput($"Bot: {response}\n");
}, () =>
{
// Speech synthesis finished.
});
});
});
});
_uiController.CurrentState = UIController.UIState.Idle;
}
private void HandleTalkButtonClicked()
{
if (!_apiManager.IsSessionActive)
{
_apiManager.InitiateSession(
() => _uiController.SessionActive = _apiManager.IsSessionActive);
Debug.LogWarning("Session is not active. Cannot send message.");
return;
}
else
{
_apiManager.CloseSession(
() => _uiController.SessionActive = _apiManager.IsSessionActive
);
}
}
private void HandleSendChatButtonClicked(string message)
{
_apiManager.SendChatMessage(message,
(responseMessage) =>
switch (_uiController.CurrentState)
{
_uiController.ChatOutput += $"User: {message}\n";
_uiController.ChatOutput += $"Bot: {responseMessage}\n";
},
() =>
{
// Speech finished callback (optional)
});
case UIController.UIState.Idle:
_audioCapture.StartRecording();
_uiController.CurrentState = UIController.UIState.Recording;
break;
case UIController.UIState.Recording:
_audioCapture.StopRecording();
_uiController.CurrentState = UIController.UIState.Idle;
break;
case UIController.UIState.Processing:
Debug.Log("Currently processing. Please wait.");
break;
}
}
}
}
Assets/_Client/Scripts/Core/AudioCapture.cs
+265
View File
@@ -0,0 +1,265 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using UnityEngine;
namespace PPGIA.X540.Project3
{
[RequireComponent(typeof(AudioSource))]
public class AudioCapture : MonoBehaviour
{
public enum SampleRate
{
Hz16000 = 16000,
Hz44100 = 44100,
Hz48000 = 48000,
Hz96000 = 96000
}
#region -- Fields & Properties ----------------------------------------
[Header("Audio Capture Settings")]
[SerializeField]
private SampleRate _sampleRateInHz = SampleRate.Hz44100;
[SerializeField]
private int _maxRecordingSeconds = 300; // safety cap
[SerializeField]
private string _fileName = "RecordedAudio.wav";
[SerializeField, Range(0f, 1f)]
private float _playbackVolume = 1f; // volume while monitoring
[SerializeField]
private bool _enableMonitoring = true; // if true, playback your mic
[SerializeField, Tooltip("Latency in milliseconds to offset monitoring playback from the microphone write head")]
private int _monitorLatencyMs = 80;
[SerializeField]
private string[] _microphones;
[SerializeField]
private int _selectedMicrophoneIndex = 0;
public string SelectedDevice =>
_microphones.Length > 0 &&
_selectedMicrophoneIndex < _microphones.Length &&
_selectedMicrophoneIndex >= 0
? _microphones[_selectedMicrophoneIndex]
: null;
public bool IsRecording => _isRecording;
public string LastSavedFilePath { get; private set; }
public event Action<string> OnRecordingSaved; // path
public event Action OnRecordingStarted;
public event Action OnRecordingStopped;
private AudioSource _audioSource;
private bool _isRecording;
private List<short> _capturedSamples =
new List<short>(1024 * 32); // filled only on Stop
private int _channels = 1; // microphone channel count (Unity usually mono)
private AudioClip _recordingClip;
public AudioClip GetRecordedClip() => _recordingClip;
private string _currentDevice;
#endregion ------------------------------------------------------------
#region -- MonoBehaviour Methods --------------------------------------
private void Awake()
{
_audioSource = GetComponent<AudioSource>();
_microphones = Microphone.devices;
}
private void OnDestroy()
{
StopRecording();
}
#endregion ------------------------------------------------------------
[ContextMenu("Start recording audio")]
public void StartRecording()
{
if (_isRecording)
{
Debug.LogWarning("Already recording.");
return;
}
if (_microphones == null || _microphones.Length == 0)
{
Debug.LogError("No microphone devices found.");
return;
}
_currentDevice = SelectedDevice;
int frequency = GetSupportedFrequency(_currentDevice, (int)_sampleRateInHz);
// start as looped for safe monitoring without underruns
_recordingClip = Microphone.Start(
_currentDevice, true, _maxRecordingSeconds, frequency);
if (_recordingClip == null)
{
Debug.LogError("Failed to start microphone recording.");
return;
}
_channels = Mathf.Max(1, _recordingClip.channels);
_capturedSamples.Clear();
_isRecording = true;
StartCoroutine(WaitAndStartMonitoring());
OnRecordingStarted?.Invoke();
}
private IEnumerator WaitAndStartMonitoring()
{
// Wait until microphone has started providing data
while (Microphone.GetPosition(_currentDevice) <= 0)
{
yield return null;
}
// Recording might have been stopped early
if (!_isRecording) yield break;
if (_enableMonitoring)
{
_audioSource.loop = true;
_audioSource.clip = _recordingClip;
_audioSource.volume = _playbackVolume;
// Start playback slightly behind the microphone write position to avoid underruns/noise
int pos = Microphone.GetPosition(_currentDevice);
int latency = Mathf.Clamp((int)(_recordingClip.frequency * (_monitorLatencyMs / 1000f)), 0, _recordingClip.samples - 1);
int start = pos - latency;
if (start < 0) start += _recordingClip.samples; // wrap around for looped clip
_audioSource.timeSamples = start % _recordingClip.samples;
_audioSource.Play();
}
}
[ContextMenu("Stop recording audio")]
public void StopRecording()
{
if (!_isRecording)
{
Debug.LogWarning("Not currently recording.");
return;
}
if (_audioSource.isPlaying) _audioSource.Stop();
var position = Microphone.GetPosition(_currentDevice);
Microphone.End(_currentDevice);
_isRecording = false;
// Capture only the recorded portion (GetPosition tells how many samples per channel were written)
if (position <= 0)
{
Debug.LogWarning("Microphone position is zero; no data captured.");
}
else
{
try
{
int samplesToCopy = position * _recordingClip.channels;
float[] floatData = new float[samplesToCopy];
_recordingClip.GetData(floatData, 0);
for (int i = 0; i < floatData.Length; i++)
{
float clamped = Mathf.Clamp(floatData[i], -1f, 1f);
short sample = (short)(clamped * short.MaxValue);
_capturedSamples.Add(sample);
}
}
catch (Exception ex)
{
Debug.LogError($"Failed to read microphone data: {ex.Message}");
}
}
OnRecordingStopped?.Invoke();
SaveWavFile();
}
private void SaveWavFile()
{
if (_capturedSamples.Count == 0)
{
Debug.LogWarning("No audio samples captured; skipping file save.");
return;
}
string filePath = Path.Combine(Application.persistentDataPath, _fileName);
try
{
using (var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write))
using (var writer = new BinaryWriter(fs))
{
int sampleRate = _recordingClip != null ? _recordingClip.frequency : (int)_sampleRateInHz;
int bitsPerSample = 16;
int channels = _recordingClip != null ? _recordingClip.channels : _channels;
int byteRate = sampleRate * channels * bitsPerSample / 8;
byte[] dataBytes = new byte[_capturedSamples.Count * 2];
Buffer.BlockCopy(_capturedSamples.ToArray(), 0, dataBytes, 0, dataBytes.Length);
int subchunk2Size = dataBytes.Length;
int chunkSize = 36 + subchunk2Size;
// RIFF header
writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"));
writer.Write(chunkSize);
writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE"));
// fmt subchunk
writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt "));
writer.Write(16); // PCM header length
writer.Write((short)1); // Audio format = PCM
writer.Write((short)channels);
writer.Write(sampleRate);
writer.Write(byteRate);
writer.Write((short)(channels * bitsPerSample / 8)); // block align
writer.Write((short)bitsPerSample);
// data subchunk
writer.Write(System.Text.Encoding.ASCII.GetBytes("data"));
writer.Write(subchunk2Size);
writer.Write(dataBytes);
}
LastSavedFilePath = filePath;
OnRecordingSaved?.Invoke(filePath);
}
catch (Exception ex)
{
Debug.LogError($"Failed to save WAV file: {ex.Message}");
}
finally
{
_capturedSamples.Clear();
}
}
[ContextMenu("Refresh microphone list")]
public void RefreshMicrophones()
{
_microphones = Microphone.devices;
if (_microphones == null || _microphones.Length == 0)
{
Debug.LogWarning("No microphones detected after refresh.");
_selectedMicrophoneIndex = -1;
}
else if (
_selectedMicrophoneIndex < 0 ||
_selectedMicrophoneIndex >= _microphones.Length)
{
_selectedMicrophoneIndex = 0;
}
}
private int GetSupportedFrequency(string device, int requested)
{
int min, max;
Microphone.GetDeviceCaps(device, out min, out max);
// When both are 0, any frequency is supported
if (min == 0 && max == 0) return requested;
if (requested < min) return min;
if (requested > max) return max;
return requested;
}
}
}
Assets/_Client/Scripts/Core/AudioCapture.cs.meta
+2
View File
@@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 9523fc2f4430eb549ab3da789eaf70c1