first commit
Nhat Nam Nguyen
parent
6ba63ad7a5
commit
d3a3de32cb
@ -1,2 +1,224 @@
|
|||||||
package com.example.awslextest;public class AudioEventsSubscription {
|
package com.example.awslextest;
|
||||||
|
|
||||||
|
import org.reactivestreams.Subscriber;
|
||||||
|
import org.reactivestreams.Subscription;
|
||||||
|
import software.amazon.awssdk.core.SdkBytes;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.AudioInputEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.ConfigurationEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.DisconnectionEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.PlaybackCompletionEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationRequestEventStream;
|
||||||
|
|
||||||
|
import javax.sound.sampled.AudioFormat;
|
||||||
|
import javax.sound.sampled.AudioInputStream;
|
||||||
|
import javax.sound.sampled.AudioSystem;
|
||||||
|
import javax.sound.sampled.DataLine;
|
||||||
|
import javax.sound.sampled.LineUnavailableException;
|
||||||
|
import javax.sound.sampled.TargetDataLine;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.UncheckedIOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.concurrent.BlockingQueue;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
public class AudioEventsSubscription implements Subscription{
|
||||||
|
private static final AudioFormat MIC_FORMAT = new AudioFormat(8000, 16, 1, true, false);
|
||||||
|
private static final String AUDIO_CONTENT_TYPE = "audio/lpcm; sample-rate=8000; sample-size-bits=16; channel-count=1; is-big-endian=false";
|
||||||
|
//private static final String RESPONSE_TYPE = "audio/pcm; sample-rate=8000";
|
||||||
|
private static final String RESPONSE_TYPE = "audio/mpeg";
|
||||||
|
private static final int BYTES_IN_AUDIO_CHUNK = 320;
|
||||||
|
private static final AtomicLong eventIdGenerator = new AtomicLong(0);
|
||||||
|
|
||||||
|
private final AudioInputStream audioInputStream;
|
||||||
|
private final Subscriber<? super StartConversationRequestEventStream> subscriber;
|
||||||
|
private final EventWriter eventWriter;
|
||||||
|
private CompletableFuture eventWriterFuture;
|
||||||
|
|
||||||
|
|
||||||
|
public AudioEventsSubscription(Subscriber<? super StartConversationRequestEventStream> subscriber) {
|
||||||
|
this.audioInputStream = getMicStream();
|
||||||
|
this.subscriber = subscriber;
|
||||||
|
this.eventWriter = new EventWriter(subscriber, audioInputStream);
|
||||||
|
configureConversation();
|
||||||
|
}
|
||||||
|
|
||||||
|
private AudioInputStream getMicStream() {
|
||||||
|
try {
|
||||||
|
DataLine.Info dataLineInfo = new DataLine.Info(TargetDataLine.class, MIC_FORMAT);
|
||||||
|
TargetDataLine targetDataLine = (TargetDataLine) AudioSystem.getLine(dataLineInfo);
|
||||||
|
|
||||||
|
targetDataLine.open(MIC_FORMAT);
|
||||||
|
targetDataLine.start();
|
||||||
|
|
||||||
|
return new AudioInputStream(targetDataLine);
|
||||||
|
} catch (LineUnavailableException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void request(long demand) {
|
||||||
|
// If a thread to write events has not been started, start it.
|
||||||
|
if (eventWriterFuture == null) {
|
||||||
|
eventWriterFuture = CompletableFuture.runAsync(eventWriter);
|
||||||
|
}
|
||||||
|
eventWriter.addDemand(demand);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void cancel() {
|
||||||
|
subscriber.onError(new RuntimeException("stream was cancelled"));
|
||||||
|
try {
|
||||||
|
audioInputStream.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new UncheckedIOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void configureConversation() {
|
||||||
|
String eventId = "ConfigurationEvent-" + String.valueOf(eventIdGenerator.incrementAndGet());
|
||||||
|
|
||||||
|
ConfigurationEvent configurationEvent = StartConversationRequestEventStream
|
||||||
|
.configurationEventBuilder()
|
||||||
|
.eventId(eventId)
|
||||||
|
.clientTimestampMillis(System.currentTimeMillis())
|
||||||
|
.responseContentType(RESPONSE_TYPE)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
System.out.println("writing config event");
|
||||||
|
eventWriter.writeConfigurationEvent(configurationEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void disconnect() {
|
||||||
|
|
||||||
|
String eventId = "DisconnectionEvent-" + String.valueOf(eventIdGenerator.incrementAndGet());
|
||||||
|
|
||||||
|
DisconnectionEvent disconnectionEvent = StartConversationRequestEventStream
|
||||||
|
.disconnectionEventBuilder()
|
||||||
|
.eventId(eventId)
|
||||||
|
.clientTimestampMillis(System.currentTimeMillis())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
eventWriter.writeDisconnectEvent(disconnectionEvent);
|
||||||
|
|
||||||
|
try {
|
||||||
|
audioInputStream.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new UncheckedIOException(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
//Notify the subscriber that we've finished.
|
||||||
|
public void stop() {
|
||||||
|
subscriber.onComplete();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void playbackFinished() {
|
||||||
|
String eventId = "PlaybackCompletion-" + String.valueOf(eventIdGenerator.incrementAndGet());
|
||||||
|
|
||||||
|
PlaybackCompletionEvent playbackCompletionEvent = StartConversationRequestEventStream
|
||||||
|
.playbackCompletionEventBuilder()
|
||||||
|
.eventId(eventId)
|
||||||
|
.clientTimestampMillis(System.currentTimeMillis())
|
||||||
|
.build();
|
||||||
|
|
||||||
|
eventWriter.writePlaybackFinishedEvent(playbackCompletionEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class EventWriter implements Runnable {
|
||||||
|
private final BlockingQueue<StartConversationRequestEventStream> eventQueue;
|
||||||
|
private final AudioInputStream audioInputStream;
|
||||||
|
private final AtomicLong demand;
|
||||||
|
private final Subscriber subscriber;
|
||||||
|
|
||||||
|
private boolean conversationConfigured;
|
||||||
|
|
||||||
|
public EventWriter(Subscriber subscriber, AudioInputStream audioInputStream) {
|
||||||
|
this.eventQueue = new LinkedBlockingQueue<>();
|
||||||
|
|
||||||
|
this.demand = new AtomicLong(0);
|
||||||
|
this.subscriber = subscriber;
|
||||||
|
this.audioInputStream = audioInputStream;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeConfigurationEvent(ConfigurationEvent configurationEvent) {
|
||||||
|
eventQueue.add(configurationEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeDisconnectEvent(DisconnectionEvent disconnectionEvent) {
|
||||||
|
eventQueue.add(disconnectionEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writePlaybackFinishedEvent(PlaybackCompletionEvent playbackCompletionEvent) {
|
||||||
|
eventQueue.add(playbackCompletionEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
void addDemand(long l) {
|
||||||
|
this.demand.addAndGet(l);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
try {
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
long currentDemand = demand.get();
|
||||||
|
|
||||||
|
if (currentDemand > 0) {
|
||||||
|
// Try to read from queue of events.
|
||||||
|
// If nothing is in queue at this point, read the audio events directly from audio stream.
|
||||||
|
for (long i = 0; i < currentDemand; i++) {
|
||||||
|
|
||||||
|
if (eventQueue.peek() != null) {
|
||||||
|
subscriber.onNext(eventQueue.take());
|
||||||
|
demand.decrementAndGet();
|
||||||
|
} else {
|
||||||
|
writeAudioEvent();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
throw new RuntimeException("interrupted when reading data to be sent to server");
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void writeAudioEvent() {
|
||||||
|
byte[] bytes = new byte[BYTES_IN_AUDIO_CHUNK];
|
||||||
|
|
||||||
|
int numBytesRead = 0;
|
||||||
|
try {
|
||||||
|
numBytesRead = audioInputStream.read(bytes);
|
||||||
|
if (numBytesRead != -1) {
|
||||||
|
byte[] byteArrayCopy = Arrays.copyOf(bytes, numBytesRead);
|
||||||
|
|
||||||
|
String eventId = "AudioEvent-" + String.valueOf(eventIdGenerator.incrementAndGet());
|
||||||
|
|
||||||
|
AudioInputEvent audioInputEvent = StartConversationRequestEventStream
|
||||||
|
.audioInputEventBuilder()
|
||||||
|
.audioChunk(SdkBytes.fromByteBuffer(ByteBuffer.wrap(byteArrayCopy)))
|
||||||
|
.contentType(AUDIO_CONTENT_TYPE)
|
||||||
|
.clientTimestampMillis(System.currentTimeMillis())
|
||||||
|
.eventId(eventId).build();
|
||||||
|
|
||||||
|
//System.out.println("sending audio event:" + audioInputEvent);
|
||||||
|
subscriber.onNext(audioInputEvent);
|
||||||
|
demand.decrementAndGet();
|
||||||
|
//System.out.println("sent audio event:" + audioInputEvent);
|
||||||
|
} else {
|
||||||
|
subscriber.onComplete();
|
||||||
|
System.out.println("audio stream has ended");
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (IOException e) {
|
||||||
|
System.out.println("got an exception when reading from audio stream");
|
||||||
|
System.err.println(e);
|
||||||
|
subscriber.onError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,2 +1,58 @@
|
|||||||
package com.example.awslextest;public class AudioResponse {
|
package com.example.awslextest;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.UncheckedIOException;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
public class AudioResponse extends InputStream{
|
||||||
|
// Used to convert byte, which is signed in Java, to positive integer (unsigned)
|
||||||
|
private static final int UNSIGNED_BYTE_MASK = 0xFF;
|
||||||
|
private static final long POLL_INTERVAL_MS = 10;
|
||||||
|
|
||||||
|
private final LinkedBlockingQueue<Integer> byteQueue = new LinkedBlockingQueue<>();
|
||||||
|
|
||||||
|
private volatile boolean closed;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int read() throws IOException {
|
||||||
|
try {
|
||||||
|
Optional<Integer> maybeInt;
|
||||||
|
while (true) {
|
||||||
|
maybeInt = Optional.ofNullable(this.byteQueue.poll(POLL_INTERVAL_MS, TimeUnit.MILLISECONDS));
|
||||||
|
|
||||||
|
// If we get an integer from the queue, return it.
|
||||||
|
if (maybeInt.isPresent()) {
|
||||||
|
return maybeInt.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the stream is closed and there is nothing queued up, return -1.
|
||||||
|
if (this.closed) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
throw new IOException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Writes data into the stream to be offered on future read() calls.
|
||||||
|
*/
|
||||||
|
public void write(byte[] byteArray) {
|
||||||
|
// Don't write into the stream if it is already closed.
|
||||||
|
if (this.closed) {
|
||||||
|
throw new UncheckedIOException(new IOException("Stream already closed when attempting to write into it."));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (byte b : byteArray) {
|
||||||
|
this.byteQueue.add(b & UNSIGNED_BYTE_MASK);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
this.closed = true;
|
||||||
|
super.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,2 +1,193 @@
|
|||||||
package com.example.awslextest;public class BotResponseHandler {
|
package com.example.awslextest;
|
||||||
|
|
||||||
|
import javazoom.jl.decoder.JavaLayerException;
|
||||||
|
import javazoom.jl.player.advanced.AdvancedPlayer;
|
||||||
|
import javazoom.jl.player.advanced.PlaybackEvent;
|
||||||
|
import javazoom.jl.player.advanced.PlaybackListener;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.core.async.SdkPublisher;
|
||||||
|
import software.amazon.awssdk.regions.Region;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.AudioResponseEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.DialogActionType;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.IntentResultEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.PlaybackInterruptionEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationResponse;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationResponseEventStream;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationResponseHandler;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.TextResponseEvent;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.TranscriptEvent;
|
||||||
|
|
||||||
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||||
|
import software.amazon.awssdk.core.sync.RequestBody;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.UncheckedIOException;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
public class BotResponseHandler implements StartConversationResponseHandler{
|
||||||
|
private final EventsPublisher eventsPublisher;
|
||||||
|
|
||||||
|
private boolean lastBotResponsePlayedBack;
|
||||||
|
private boolean isDialogStateClosed;
|
||||||
|
private AudioResponse audioResponse;
|
||||||
|
|
||||||
|
private String sessionId;
|
||||||
|
private TextResponseEvent textResponseEvent;
|
||||||
|
private SoundRecorder recorder;
|
||||||
|
private String bucketName;
|
||||||
|
|
||||||
|
private S3Client s3;
|
||||||
|
public BotResponseHandler(EventsPublisher eventsPublisher,String sessionId) {
|
||||||
|
this.eventsPublisher = eventsPublisher;
|
||||||
|
this.lastBotResponsePlayedBack = false;// At the start, we have not played back last response from bot.
|
||||||
|
this.isDialogStateClosed = false; // At the start, the dialog state is open.
|
||||||
|
this.sessionId = sessionId;
|
||||||
|
|
||||||
|
this.bucketName = "demo-bucket-lex";
|
||||||
|
String accessKey = System.getenv("AWS_ACCESS_KEY_ID");
|
||||||
|
String secretKey = System.getenv("AWS_SECRET_KEY");
|
||||||
|
AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
|
||||||
|
.create(AwsBasicCredentials.create(accessKey, secretKey));
|
||||||
|
this.s3 = S3Client.builder()
|
||||||
|
.region(Region.AWS_GLOBAL)
|
||||||
|
.credentialsProvider(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
this.recorder = new SoundRecorder(this.sessionId);
|
||||||
|
recorder.bootUp();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void responseReceived(StartConversationResponse startConversationResponse) {
|
||||||
|
System.out.println("successfully established the connection with server. request id:" + startConversationResponse.responseMetadata().requestId()); // would have 2XX, request id.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onEventStream(SdkPublisher<StartConversationResponseEventStream> sdkPublisher) {
|
||||||
|
|
||||||
|
sdkPublisher.subscribe(event -> {
|
||||||
|
if (event instanceof PlaybackInterruptionEvent) {
|
||||||
|
handle((PlaybackInterruptionEvent) event);
|
||||||
|
} else if (event instanceof TranscriptEvent) {
|
||||||
|
handle((TranscriptEvent) event);
|
||||||
|
} else if (event instanceof IntentResultEvent) {
|
||||||
|
handle((IntentResultEvent) event);
|
||||||
|
} else if (event instanceof TextResponseEvent) {
|
||||||
|
handle((TextResponseEvent) event);
|
||||||
|
} else if (event instanceof AudioResponseEvent) {
|
||||||
|
handle((AudioResponseEvent) event);
|
||||||
|
}
|
||||||
|
//else if (event instanceof AudioInputEvent) {
|
||||||
|
// handle((AudioInputEvent) event);
|
||||||
|
// }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void exceptionOccurred(Throwable throwable) {
|
||||||
|
System.err.println("got an exception:" + throwable);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void complete() {
|
||||||
|
System.out.println("on complete");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void handle(PlaybackInterruptionEvent event) {
|
||||||
|
System.out.println("Got a PlaybackInterruptionEvent: " + event);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void handle(TranscriptEvent event) {
|
||||||
|
System.out.println("Got a TranscriptEvent: " + event);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void handle(IntentResultEvent event) {
|
||||||
|
System.out.println("Got an IntentResultEvent: " + event);
|
||||||
|
isDialogStateClosed = DialogActionType.CLOSE.equals(event.sessionState().dialogAction().type());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void handle(TextResponseEvent event) {
|
||||||
|
this.textResponseEvent = event;
|
||||||
|
System.out.println("Got an TextResponseEvent: " + event);
|
||||||
|
event.messages().forEach(message -> {
|
||||||
|
System.out.println("Message content type:" + message.contentType());
|
||||||
|
System.out.println("Message content:" + message.content());
|
||||||
|
});
|
||||||
|
if (this.textResponseEvent.messages().get(0).content().equals("Please say I, henry, hereby sign this document.")){
|
||||||
|
CompletableFuture.runAsync(() -> {
|
||||||
|
this.recorder.start();
|
||||||
|
PutObjectRequest objectRequest = PutObjectRequest.builder()
|
||||||
|
.bucket(this.bucketName)
|
||||||
|
.key(String.format("%s.wav",this.sessionId))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
this.s3.putObject(objectRequest, RequestBody.fromFile(this.recorder.getWavFile()));
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
this.recorder.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void handle(AudioResponseEvent event) {//Synthesize speech
|
||||||
|
// System.out.println("Got a AudioResponseEvent: " + event);
|
||||||
|
if (audioResponse == null) {
|
||||||
|
audioResponse = new AudioResponse();
|
||||||
|
//Start an audio player in a different thread.
|
||||||
|
CompletableFuture.runAsync(() -> {
|
||||||
|
try {
|
||||||
|
AdvancedPlayer audioPlayer = new AdvancedPlayer(audioResponse);
|
||||||
|
|
||||||
|
audioPlayer.setPlayBackListener(new PlaybackListener() {
|
||||||
|
@Override
|
||||||
|
public void playbackFinished(PlaybackEvent evt) {
|
||||||
|
super.playbackFinished(evt);
|
||||||
|
|
||||||
|
// Inform the Amazon Lex bot that the playback has finished.
|
||||||
|
eventsPublisher.playbackFinished();
|
||||||
|
if (isDialogStateClosed) {
|
||||||
|
lastBotResponsePlayedBack = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
audioPlayer.play();
|
||||||
|
} catch (JavaLayerException e) {
|
||||||
|
throw new RuntimeException("got an exception when using audio player", e);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (event.audioChunk() != null) {
|
||||||
|
audioResponse.write(event.audioChunk().asByteArray());
|
||||||
|
} else {
|
||||||
|
// The audio prompt has ended when the audio response has no
|
||||||
|
// audio bytes.
|
||||||
|
try {
|
||||||
|
audioResponse.close();
|
||||||
|
audioResponse = null; // Prepare for the next audio prompt.
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new UncheckedIOException("got an exception when closing the audio response", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// private void handle(AudioInputEvent event){
|
||||||
|
// this.audioInputEvent = event;
|
||||||
|
// System.out.println("Got a AudioInputEvent: ");
|
||||||
|
// if (this.textResponseEvent.messages().get(0).content() == "") {
|
||||||
|
// CompletableFuture.runAsync(() -> {
|
||||||
|
// event.audioChunk();
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// The conversation with the Amazon Lex bot is complete when the bot marks the Dialog as DialogActionType.CLOSE
|
||||||
|
// and any prompt playback is finished. For more information, see
|
||||||
|
// https://docs.aws.amazon.com/lexv2/latest/dg/API_runtime_DialogAction.html.
|
||||||
|
public boolean isConversationComplete() {
|
||||||
|
return isDialogStateClosed && lastBotResponsePlayedBack;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,2 +1,38 @@
|
|||||||
package com.example.awslextest;public class EventsPublisher {
|
package com.example.awslextest;
|
||||||
|
|
||||||
|
import org.reactivestreams.Publisher;
|
||||||
|
import org.reactivestreams.Subscriber;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationRequestEventStream;
|
||||||
|
public class EventsPublisher implements Publisher<StartConversationRequestEventStream>{
|
||||||
|
private AudioEventsSubscription audioEventsSubscription;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void subscribe(Subscriber<? super StartConversationRequestEventStream> subscriber) {
|
||||||
|
if (audioEventsSubscription == null) {
|
||||||
|
|
||||||
|
audioEventsSubscription = new AudioEventsSubscription(subscriber);
|
||||||
|
subscriber.onSubscribe(audioEventsSubscription);
|
||||||
|
|
||||||
|
} else {
|
||||||
|
throw new IllegalStateException("received unexpected subscription request");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void disconnect() {
|
||||||
|
if (audioEventsSubscription != null) {
|
||||||
|
audioEventsSubscription.disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
if (audioEventsSubscription != null) {
|
||||||
|
audioEventsSubscription.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void playbackFinished() {
|
||||||
|
if (audioEventsSubscription != null) {
|
||||||
|
audioEventsSubscription.playbackFinished();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,2 +1,96 @@
|
|||||||
package com.example.awslextest;public class LexBidirectionalStreamingExample {
|
package com.example.awslextest;
|
||||||
|
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.regions.Region;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.LexRuntimeV2AsyncClient;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.ConversationMode;
|
||||||
|
import software.amazon.awssdk.services.lexruntimev2.model.StartConversationRequest;
|
||||||
|
|
||||||
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
|
||||||
|
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.util.UUID;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
|
||||||
|
public class LexBidirectionalStreamingExample {
|
||||||
|
public static void main(String[] args) throws URISyntaxException, InterruptedException {
|
||||||
|
String botId = System.getenv("AWS_LEX_BOT_ID ");
|
||||||
|
String botAliasId = System.getenv("AWS_LEX_BOT_ALIAS_ID");
|
||||||
|
String localeId = "en_US";
|
||||||
|
String accessKey = System.getenv("AWS_ACCESS_KEY_ID");
|
||||||
|
String secretKey = System.getenv("AWS_SECRET_KEY");
|
||||||
|
String sessionId = UUID.randomUUID().toString();
|
||||||
|
Region region = Region.US_EAST_1; // Choose an AWS Region where the Amazon Lex Streaming API is available.
|
||||||
|
|
||||||
|
AwsCredentialsProvider awsCredentialsProvider = StaticCredentialsProvider
|
||||||
|
.create(AwsBasicCredentials.create(accessKey, secretKey));
|
||||||
|
|
||||||
|
// Create a new SDK client. You need to use an asynchronous client.
|
||||||
|
System.out.println("step 1: creating a new Lex SDK client");
|
||||||
|
LexRuntimeV2AsyncClient lexRuntimeServiceClient = LexRuntimeV2AsyncClient.builder()
|
||||||
|
.region(region)
|
||||||
|
.credentialsProvider(awsCredentialsProvider)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
|
||||||
|
// Configure the bot, alias and locale that you'll use to have a conversation.
|
||||||
|
System.out.println("step 2: configuring bot details");
|
||||||
|
StartConversationRequest.Builder startConversationRequestBuilder = StartConversationRequest.builder()
|
||||||
|
.botId(botId)
|
||||||
|
.botAliasId(botAliasId)
|
||||||
|
.localeId(localeId);
|
||||||
|
|
||||||
|
// Configure the conversation mode of the bot. By default, the
|
||||||
|
// conversation mode is audio.
|
||||||
|
System.out.println("step 3: choosing conversation mode");
|
||||||
|
startConversationRequestBuilder = startConversationRequestBuilder.conversationMode(ConversationMode.AUDIO);
|
||||||
|
|
||||||
|
// Assign a unique identifier for the conversation.
|
||||||
|
System.out.println("step 4: choosing a unique conversation identifier");
|
||||||
|
startConversationRequestBuilder = startConversationRequestBuilder.sessionId(sessionId);
|
||||||
|
|
||||||
|
// Start the initial request.
|
||||||
|
StartConversationRequest startConversationRequest = startConversationRequestBuilder.build();
|
||||||
|
|
||||||
|
// Create a stream of audio data to the Amazon Lex bot. The stream will start after the connection is established with the bot.
|
||||||
|
EventsPublisher eventsPublisher = new EventsPublisher();
|
||||||
|
|
||||||
|
// Create a class to handle responses from bot. After the server processes the user data you've streamed, the server responds
|
||||||
|
// on another stream.
|
||||||
|
BotResponseHandler botResponseHandler = new BotResponseHandler(eventsPublisher,sessionId);
|
||||||
|
|
||||||
|
// Start a connection and pass in the publisher that streams the audio and process the responses from the bot.
|
||||||
|
System.out.println("step 5: starting the conversation ...");
|
||||||
|
CompletableFuture<Void> conversation = lexRuntimeServiceClient.startConversation(
|
||||||
|
startConversationRequest,
|
||||||
|
eventsPublisher,
|
||||||
|
botResponseHandler
|
||||||
|
);
|
||||||
|
|
||||||
|
// Wait until the conversation finishes. The conversation finishes if the dialog state reaches the "Closed" state.
|
||||||
|
// The client stops the connection. If an exception occurs during the conversation, the
|
||||||
|
// client sends a disconnection event.
|
||||||
|
conversation.whenComplete((result, exception) -> {
|
||||||
|
if (exception != null) {
|
||||||
|
eventsPublisher.disconnect();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// The conversation finishes when the dialog state is closed and last prompt has been played.
|
||||||
|
while (!botResponseHandler.isConversationComplete()) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Randomly sleep for 100 milliseconds to prevent JVM from exiting.
|
||||||
|
// You won't need this in your production code because your JVM is
|
||||||
|
// likely to always run.
|
||||||
|
// When the conversation finishes, the following code block stops publishing more data and informs the Amazon Lex bot that there is no more data to send.
|
||||||
|
if (botResponseHandler.isConversationComplete()) {
|
||||||
|
System.out.println("conversation is complete.");
|
||||||
|
eventsPublisher.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,2 +1,197 @@
|
|||||||
package com.example.awslextest;public class SoundRecorder {
|
package com.example.awslextest;
|
||||||
|
import javax.sound.sampled.*;
|
||||||
|
import java.io.*;
|
||||||
|
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||||
|
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||||
|
import software.amazon.awssdk.core.waiters.WaiterResponse;
|
||||||
|
import software.amazon.awssdk.regions.Region;
|
||||||
|
import software.amazon.awssdk.services.s3.S3Client;
|
||||||
|
import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
|
||||||
|
import software.amazon.awssdk.core.sync.RequestBody;
|
||||||
|
import software.amazon.awssdk.services.s3.model.S3Exception;
|
||||||
|
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
|
||||||
|
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
|
||||||
|
import software.amazon.awssdk.services.s3.model.S3Object;
|
||||||
|
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CompletedPart;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
|
||||||
|
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.waiters.S3Waiter;
|
||||||
|
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
|
||||||
|
import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
|
||||||
|
/*
|
||||||
|
Define an audio format of the sound source to be captured, using the class AudioFormat.
|
||||||
|
Create a DataLine.Info object to hold information of a data line.
|
||||||
|
Obtain a TargetDataLine object which represents an input data line from which audio data can be captured, using the method getLineInfo(DataLine.Info) of the AudioSystem class.
|
||||||
|
Open and start the target data line to begin capturing audio data.
|
||||||
|
Create an AudioInputStream object to read data from the target data line.
|
||||||
|
Record the captured sound into a WAV file using the following method of the class AudioSystem:
|
||||||
|
write(AudioInputStream, AudioFileFormat.Type, File)
|
||||||
|
Note that this method blocks the current thread until the target data line is closed.
|
||||||
|
|
||||||
|
Stop and close the target data line to end capturing and recording.
|
||||||
|
|
||||||
|
*/
|
||||||
|
public class SoundRecorder {
|
||||||
|
// record duration, in milliseconds
|
||||||
|
static final long RECORD_TIME = 10000; // 1 minute
|
||||||
|
|
||||||
|
// path of the wav file
|
||||||
|
private File wavFile;
|
||||||
|
|
||||||
|
// format of audio file
|
||||||
|
AudioFileFormat.Type fileType = AudioFileFormat.Type.WAVE;
|
||||||
|
|
||||||
|
// the line from which audio data is captured
|
||||||
|
TargetDataLine line;
|
||||||
|
AudioFormat format;
|
||||||
|
private String sessionId;
|
||||||
|
|
||||||
|
private S3Client s3;
|
||||||
|
private static final Region region = Region.AWS_GLOBAL;
|
||||||
|
private String bucketName = "demo-bucket-lex";
|
||||||
|
|
||||||
|
// private ArrayList<CompletedPart> completedParts;
|
||||||
|
// private int completedPartsCount;
|
||||||
|
// private String uploadId;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
SoundRecorder(String sessionId){
|
||||||
|
|
||||||
|
this.wavFile = new File(String.format("voice_sig_files/%s.wav",sessionId));
|
||||||
|
this.sessionId = sessionId;
|
||||||
|
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Defines an audio format
|
||||||
|
*/
|
||||||
|
AudioFormat getAudioFormat() {
|
||||||
|
float sampleRate = 8000;
|
||||||
|
int sampleSizeInBits = 16;
|
||||||
|
int channels = 1;
|
||||||
|
boolean signed = true;
|
||||||
|
boolean bigEndian = false;
|
||||||
|
AudioFormat format = new AudioFormat(sampleRate, sampleSizeInBits,
|
||||||
|
channels, signed, bigEndian);
|
||||||
|
return format;
|
||||||
|
}
|
||||||
|
|
||||||
|
File getWavFile(){
|
||||||
|
return this.wavFile;
|
||||||
|
}
|
||||||
|
void bootUp(){
|
||||||
|
try {
|
||||||
|
this.format = getAudioFormat();
|
||||||
|
DataLine.Info info = new DataLine.Info(TargetDataLine.class, this.format);
|
||||||
|
|
||||||
|
// checks if system supports the data line
|
||||||
|
if (!AudioSystem.isLineSupported(info)) {
|
||||||
|
System.out.println("Line not supported");
|
||||||
|
System.exit(0);
|
||||||
|
}
|
||||||
|
this.line = (TargetDataLine) AudioSystem.getLine(info);
|
||||||
|
} catch (LineUnavailableException ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Captures the sound and record into a WAV file
|
||||||
|
*/
|
||||||
|
void start() {
|
||||||
|
try {
|
||||||
|
this.line.open(this.format);
|
||||||
|
this.line.start(); // start capturing
|
||||||
|
|
||||||
|
System.out.println("Start capturing user's voice signature...");
|
||||||
|
|
||||||
|
AudioInputStream ais = new AudioInputStream(this.line);
|
||||||
|
|
||||||
|
System.out.println("Start recording user's voice signature...");
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
AudioSystem.write(ais, fileType, wavFile);
|
||||||
|
|
||||||
|
|
||||||
|
} catch (IOException ioe) {
|
||||||
|
ioe.printStackTrace();
|
||||||
|
}
|
||||||
|
catch (LineUnavailableException ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes the target data line to finish capturing and recording
|
||||||
|
*/
|
||||||
|
void finish() {
|
||||||
|
//line.stop();
|
||||||
|
line.close();
|
||||||
|
System.out.println("Not recording user's voice signature currently.");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private static ByteBuffer getRandomByteBuffer(int size) throws IOException {
|
||||||
|
byte[] b = new byte[size];
|
||||||
|
new Random().nextBytes(b);
|
||||||
|
return ByteBuffer.wrap(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entry to run the program
|
||||||
|
*/
|
||||||
|
public static void main(String[] args) {
|
||||||
|
SoundRecorder recorder = new SoundRecorder("4");
|
||||||
|
|
||||||
|
Thread stopper = new Thread(new Runnable() {
|
||||||
|
public void run() {
|
||||||
|
try {
|
||||||
|
Thread.sleep(RECORD_TIME);
|
||||||
|
} catch (InterruptedException ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
}
|
||||||
|
recorder.finish();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
stopper.start();
|
||||||
|
|
||||||
|
// start recording
|
||||||
|
recorder.bootUp();
|
||||||
|
recorder.start();
|
||||||
|
// creates a new thread that waits for a specified
|
||||||
|
// of time before stopping
|
||||||
|
// Thread stopper = new Thread(new Runnable() {
|
||||||
|
// public void run() {
|
||||||
|
// try {
|
||||||
|
// Thread.sleep(RECORD_TIME);
|
||||||
|
// } catch (InterruptedException ex) {
|
||||||
|
// ex.printStackTrace();
|
||||||
|
// }
|
||||||
|
// recorder.finish();
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
//
|
||||||
|
// stopper.start();
|
||||||
|
//
|
||||||
|
// // start recording
|
||||||
|
// recorder.bootUp();
|
||||||
|
// recorder.start();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue